From 260f93bdd2db42483eb18ce68544380bcc6cd792 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 26 Jan 2024 11:16:03 -0600 Subject: [PATCH 01/32] started the burn developer book --- contributor-book/.gitignore | 18 +++ contributor-book/.prettierrc.json | 4 + contributor-book/LICENSE-APACHE | 1 + contributor-book/LICENSE-MIT | 1 + contributor-book/book.toml | 13 ++ contributor-book/src/SUMMARY.md | 6 + .../setting-up-the-environment.md | 1 + .../src/basic-workflow/testing.md | 1 + .../src/frequently-encountered-issues.md | 1 + .../issues-while-adding-ops.md | 28 +++++ .../guides/adding-a-new-operation-to-burn.md | 119 ++++++++++++++++++ contributor-book/src/how-to-read-this-book.md | 14 +++ contributor-book/src/overview.md | 17 +++ 13 files changed, 224 insertions(+) create mode 100644 contributor-book/.gitignore create mode 100644 contributor-book/.prettierrc.json create mode 120000 contributor-book/LICENSE-APACHE create mode 120000 contributor-book/LICENSE-MIT create mode 100644 contributor-book/book.toml create mode 100644 contributor-book/src/SUMMARY.md create mode 100644 contributor-book/src/basic-workflow/setting-up-the-environment.md create mode 100644 contributor-book/src/basic-workflow/testing.md create mode 100644 contributor-book/src/frequently-encountered-issues.md create mode 100644 contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md create mode 100644 contributor-book/src/guides/adding-a-new-operation-to-burn.md create mode 100644 contributor-book/src/how-to-read-this-book.md create mode 100644 contributor-book/src/overview.md diff --git a/contributor-book/.gitignore b/contributor-book/.gitignore new file mode 100644 index 0000000000..409ff3eb07 --- /dev/null +++ b/contributor-book/.gitignore @@ -0,0 +1,18 @@ +target + +# MacOS temp file +.DS_Store + +book-test +guide/book + +.vscode +tests/burn-book/book/ +book/ + +# Ignore Jetbrains specific files. +.idea/ + +# Ignore Vim temporary and swap files. +*.sw? +*~ \ No newline at end of file diff --git a/contributor-book/.prettierrc.json b/contributor-book/.prettierrc.json new file mode 100644 index 0000000000..d410551a92 --- /dev/null +++ b/contributor-book/.prettierrc.json @@ -0,0 +1,4 @@ +{ + "printWidth": 100, + "proseWrap": "always" +} \ No newline at end of file diff --git a/contributor-book/LICENSE-APACHE b/contributor-book/LICENSE-APACHE new file mode 120000 index 0000000000..965b606f33 --- /dev/null +++ b/contributor-book/LICENSE-APACHE @@ -0,0 +1 @@ +../LICENSE-APACHE \ No newline at end of file diff --git a/contributor-book/LICENSE-MIT b/contributor-book/LICENSE-MIT new file mode 120000 index 0000000000..76219eb72e --- /dev/null +++ b/contributor-book/LICENSE-MIT @@ -0,0 +1 @@ +../LICENSE-MIT \ No newline at end of file diff --git a/contributor-book/book.toml b/contributor-book/book.toml new file mode 100644 index 0000000000..4cf5f1f7d8 --- /dev/null +++ b/contributor-book/book.toml @@ -0,0 +1,13 @@ +[book] +authors = [ + "Wouter Doppenberg", + "Nathaniel Simard", + "Louis Fortier-Dubois", + "Dilshod Tadjibaev", + "Guillaume Lagrange", + "Joshua Ferguson" +] +language = "en" +multilingual = false +src = "src" +title = "The Burn Developer Book 🔥" diff --git a/contributor-book/src/SUMMARY.md b/contributor-book/src/SUMMARY.md new file mode 100644 index 0000000000..f71e6b4069 --- /dev/null +++ b/contributor-book/src/SUMMARY.md @@ -0,0 +1,6 @@ +- [Overview](./overview.md) +- [How to Read This Book](./how-to-read-this-book.md) +- [setting up the environment](./basic-workflow/setting-up-the-environment.md) +- [testing](./basic-workflow/testing.md) +- [adding a new operation to burn](./guides/adding-a-new-operation-to-burn.md) +- [Frequently Encountered Issues](./frequently-encountered-issues.md) \ No newline at end of file diff --git a/contributor-book/src/basic-workflow/setting-up-the-environment.md b/contributor-book/src/basic-workflow/setting-up-the-environment.md new file mode 100644 index 0000000000..5a009c6e25 --- /dev/null +++ b/contributor-book/src/basic-workflow/setting-up-the-environment.md @@ -0,0 +1 @@ +# setting up the environment diff --git a/contributor-book/src/basic-workflow/testing.md b/contributor-book/src/basic-workflow/testing.md new file mode 100644 index 0000000000..5697da2de5 --- /dev/null +++ b/contributor-book/src/basic-workflow/testing.md @@ -0,0 +1 @@ +# testing diff --git a/contributor-book/src/frequently-encountered-issues.md b/contributor-book/src/frequently-encountered-issues.md new file mode 100644 index 0000000000..ce63065686 --- /dev/null +++ b/contributor-book/src/frequently-encountered-issues.md @@ -0,0 +1 @@ +# Frequently Encountered Issues diff --git a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md new file mode 100644 index 0000000000..1a173d50b1 --- /dev/null +++ b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md @@ -0,0 +1,28 @@ +## Issues encountered while adding ops + +Most issues were fairly straight forward, and I was able to play whack-a-bug until I hit ones were I needed outside help (on the discord channels), so I'll share some of the ones that had me stumped enough to ask for help in case you too encounter them. + +### Off by .000001 errors + +```sh +---- fusion::base::tests::maxmin::tests::test_mean_dim_2d stdout ---- thread 'fusion::base::tests::maxmin::tests::test_mean_dim_2d' panicked at burn-wgpu/src/fusion/base.rs:185:5: assertion `left == right` failed left: Data { value: [1.0, 4.0], shape: Shape { dims: [2, 1] } } right: Data { value: [0.99999994, 3.9999998], shape: Shape { dims: [2, 1] } } ---- + +tests::maxmin::tests::test_mean_dim_2d stdout ---- thread 'tests::maxmin::tests::test_mean_dim_2d' panicked at burn-wgpu/src/lib.rs:49:5: assertion `left == right` failed left: Data { value: [1.0, 4.0], shape: Shape { dims: [2, 1] } } right: Data { value: [0.99999994, 3.9999998], shape: Shape { dims: [2, 1] } } +``` + +If you encounter this, swap out the `assert_eq!` in the failing test for `tensor1.assert_approx_eq` + +### Mismatched types and missing functions + +```sh +error[E0308]: mismatched types --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow.rs:48:45 | 48 | let pow1_out1 = input1.clone().powf(input1); | ---- ^^^^^^ expected `f32`, found `Tensor` | | | arguments to this method are incorrect | = note: expected type `f32` found struct `Tensor` + +note: method defined here --> {burn_dir}/burn-tensor/src/tensor/api/float.rs:65:12 | 65 | pub fn powf(self, value: f32) -> Self { | ^^^^ + +error[E0599]: no method named `powf_scalar` found for struct `Tensor` in the current scope --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow.rs:50:35 | 50 | let pow2_out1 = pow1_out1.powf_scalar(cast1_out1); | ^^^^^^^^^^^ method not found in `Tensor` + +error[E0599]: no method named `powi` found for struct `Tensor` in the current scope --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow_int.rs:49:40 | 49 | let pow1_out1 = input1.clone().powi(input1); | ^^^^ method not found in `Tensor` Some errors have detailed explanations: E0308, E0599. +For more information about an error, try `rustc --explain E0308`. error: could not compile `onnx-tests` (test "onnx_tests") due to 3 previous errors +``` + +So if you are getting this, you probably didn't impl your operator for the actual Tensor struct. I wasn't aware of the existence or role of `burn-tensor/src/tensor/api/numeric.rs`, and had first defined just the Ops for the Int and Float tensors, that coupled with `powf` existing prior to the PR though for scalar values(which had been renamed, just not in the right place), led to this confusing issue where it looked like the function was found, but the type was wrong. If that's the case, make sure that it's implemented for the appropriate type, in this case `Float` under [burn-tensor/src/tensor/api/numeric.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/api/numeric.rs#L2186), and calling the `TensorOp.foo_op` defined under [burn-tensor/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/ops/tensor.rs#L873) diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md new file mode 100644 index 0000000000..259638e40c --- /dev/null +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -0,0 +1,119 @@ + +Let's discuss how one might go about adding new operators to burn, using the example of the recently added [pow operator](https://github.com/tracel-ai/burn/pull/1133/files) (as of 01/24/2024). In that PR, the following things took place (albeit not in this order) + +## Adding the Op to burn-tensor + +burn-tensor is the crate that defines all tensor operations that need to be implemented by the various backends. The core of this lies in `burn-tensor/src/tensor/api/numeric.rs`, which is home to the numeric trait and it's implementation for the different ensor types. The implementations for the `Foo` kind call the op defined under `tensor/src/tensor/op/foo_tensor.rs`Which defines ops specific to that type for the backend. + +here is where pow was added to [burn-tensor/src/tensor/api/numeric.rs](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1618) + +1. for the [`Tensor` struct](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L553) +2. for the [numeric trait](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1618) +3. for the impl of numeric for [float](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L2186) and [int](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1903) + +Tensor is a struct that has a single member: `primitive`(defined [here](https://github.com/tracel-ai/burn/blob/main/burn-tensor/src/tensor/api/base.rs)), That is defined by it's [`Kind`](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/kind.rs#L15): one of `Bool`, `Float`, or `Int` (those linked in 3). These call the ops for that data type defined in the [`Backend`](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/backend/base.rs#L52) supertrait[^1]. This is the trait that is then implemented by the different `burn-` back-ends (such as `burn-ndarray` and `burn-wgpu`) which implement the functions if no default is provided. + +In this case, we only don't need to worry about `Bool` Tensors. Ops for `Float` is implemented under [burn-tensor/src/tensor/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/ops/tensor.rs#L873), and for `Int` under [`burn-tensor/src/tensor/ops/int_tensor.rs`](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-tensor/src/tensor/ops/int_tensor.rs#L486). The current convention is for int and bool ops, to be prefixed with `int` or `bool`, respectively. As we need to use floating point powers to ensure precision, the `powi` functions are given a default implementation that converts the right hand side of the operator to a float. + +The `Int` Tensor function use the ones defined for Float with 2 extra cast (LHS to a `Float` tensor, Output to an `Int`). Given that the rest of the code will only look at the float implementations. + +### Adding Test + +Additional Test should be added to `burn-tensor` under `burn-tensor/src/tests/ops/{op_name}.rs`(I would link to a file but I just realized I forgot to add one for non scalar pow ops). this file should then be added to `burn-tensor/src/tests/mod.rs` using the testgen macro. This test is then ran on every backend, save for those that require specific testing such as burn-autodiff + +## Adding the Op to the burn-autodiff + +Since this is probably the hardest and the least straightforward, we'll cover this backend separately. Burn-autodiff enables other backends to use autodifferentiation[^2]. Ops for float types are implemented in [burn-autodiff/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-autodiff/src/ops/tensor.rs#L1523) and need to: + +1. define a unit struct [^3] that implements a backward (pass) function +2. Within the backward function, as this is an elementwise binary operation it implements the binary function (from backward.rs under the same directory), the last 2 arguments are two closures that define the left and right partial derivatives. +3. Then defines what happens when a specific operation is tracked or untracked, where untracked just calls the function in the normal way, and tracked executes the backward function defined above + +steps 1 and 3 are boilerplate, so much so that you can probably just copy the contents of another op of the same type (binary, unary) and change the name of the struct, and ensure that either both sides have the data they need (if they need to have a copy of the opposite sided tensor, clone it's contents). + +now for step 2. Since a significant number of the people reading this probably haven't touched calculus either ever, or since however long ago you took the appropriate course, I'll assume that you, the observer, have some prior knowledge of calculus but would benefit from a review of the concepts. If this is not the case, I apologize, you can probably skim this section. + +In the case of pow, since this is a binary operation, the left and right functions are the partial derivatives with respect to the left and right sided tensors. + +Let's define the operator as a function $f(x,y)=x^{y}$, where $x$ is the left hand tensor and $y$ is the right handed tensor. The two closures are defining the partial derivatives of $f$ with respect to $x$,$y$. The eli5 is treat the other variable as a constant + +$$\frac{\delta }{\delta x} (x^{y})= y \cdot x^{y-1}$$ +is the left handed closure, and + +$$\frac{\delta }{\delta y} (x^{y}) = x^{y} \cdot ln(x)$$ + +is the right. If you aren't sure how to calculate these by hand, I recommend using [symbolab](https://www.symbolab.com/solver/partial-derivative-calculator/%5Cfrac%7B%5Cpartial%7D%7B%5Cpartial%20x%7D%5Cleft(x%5E%7By%7D%5Cright)?or=input), plug in your operator in terms of $x$ and $y$, and just swap out the variable $x$|$y$ in the partial derivative to get the other side. + +### Testing autodiff + +Test for autodiff go under [burn-autodiff/src/tests/foo.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replacing foo for whatever makes sense for your op), and for tensor operations both the left and right side need to be verified. The easiest way to do this, is to + +1. use small tensors with simple values +2. pop open a terminal and launch `ipython` import `numpy` (or just use [google colab](https://colab.google/) if you don't have the packages installed and don't want to install them), and do the calculations by hand. +3. comparing the actual to expected output for lhs, rhs and regular operation + +generally, it seems preferable to use `actual_output_tensor.to_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. + +## Adding the Op to other backends + +most of these are fairly straightforward implementations. for reference here's pow's float implementation for torch, ndarray and candle backends: + +1. Torch implementation in [burn-tch/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/main/burn-tch/src/ops/tensor.rs#L461) and the Op used in [burn-tch/src/ops/base.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tch/src/ops/base.rs#L443) +2. NdArray in [burn-ndarray/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/main/burn-ndarray/src/ops/tensor.rs#L443) +3. Candle in [burn-candle/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-candle/src/ops/tensor.rs#L481) + +This is where any calculation happens currently. Playing a guessing game with method names and seeing what completions are suggested will take you far. If you are having trouble figuring out how to do it from the docs for that backend, [try searching github for relevant function calls](https://docs.github.com/en/search-github/github-code-search/understanding-github-code-search-syntax). + +## Adding the Op to fusion and wgpu backends + +Adding an operator to these backends is fairly straightforward, though due to what these backends are for, involves a bit more indirection. Fusion, like autodiff, is not a target backend as much as a backend that enables certain functionality for other backends, in this case kernel fusion (which is currently only supported for `burn-wgpu`), so adding the operator won't involve doing any calculation, you'll just be describing how the generated code should look. Most of this can be copy/pasted/adjusted from other functions. + +here's how powf was added to burn fusion: + +1. added powf to the float ops under [burn-fusion/src/ops/float.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/ops/float.rs#L1758) +2. added powf to the `FloatOperationDescription` enum under [burn-fusion/src/stream/operation.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/stream/operation.rs#L385) +3. added powf to the implementations of `FloatOperationDescription` enum under [burn-fusion/src/stream/context.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/stream/context.rs#L726) + + Adding pow to wgpu was actually pretty easy due to the design. Element-wise tensor ops are just vectorized scalar ops, and given that raising a tensor to a scalar power prior to the tensor version, I just reused the code for scalar powf. + +here is where code was added + +1. to the implementation of [`TensorOps` under `burn-wgpu/src/ops/float_ops.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/float_ops.rs#L513) +2. the function being called was added to [burn-wgpu/src/ops/numeric.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/numeric.rs#L199) + +There would have been a few other places additions would be necessary if some of the code for powf hadn't already been defined, such as [`burn-wgpu/src/fusion/elemwise/builder.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/fusion/elemwise/optimization.rs) and [`burn-wgpu/src/fusion/elemwise/builder.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/fusion/elemwise/builder.rs#L276), but I don't know if there are other places within `burn-wgpu` where additions would be necessary. + +## Adding the Op to burn-import + +I won't comment on generating the test onnx files or the test, as that is already covered [here](https://github.com/tracel-ai/burn/blob/main/burn-import/DEVELOPMENT.md#adding-new-operators), this is more about the specific changes you need to make when adding new operators after you have generated the tests. + +The crate is divided into two sections `src/burn` and `src/onnx`. The code under the former corresponds to the operation you've implemented earlier in this guide, and the latter to the operations defined in the onnx specification. So when you are loading a model, the operator is first parsed to an intermediate representation defined by `src/onnx`, and then mapped to a Burn operations defined under `src/burn/node`. + +Let's review the changes made for pow starting from `src/burn` and moving to `src/onnx`: + +1. determine the type of operator and add your operator to the appropriate node (operation) type, in this case [BinaryNode under burn-import/src/burn/node/binary.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/burn/node/binary.rs#L160) along with its [`to_str` definition](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/burn/node/binary.rs#L9) +2. add an arm to the match statement inside the `into_burn` function in [burn-import/src/onnx/to_burn.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/to_burn.rs#L269) for the onnx `NodeType`(corresponds to an op in the Onnx spec), and make a [`foo_conversion` function](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/to_burn.rs#L667) that maps the onnx node to the binary type +3. specify how dimensions for the output should be derived in [burn-import/src/onnx/dim_inference.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/dim_inference.rs#L103) + +And you're done! Congrats, you just fully added a new op to burn, and we are all one step closer to the answer to [are we learning yet?](https://www.arewelearningyet.com/) being "Yes, and it's freaking fast!". Buy yourself a coffee + +## Making sure everything works + +This is definitely covered elsewhere but I'd run this prior to any commit + +1. `cargo clippy --fix --allow-dirty` +2. `cargo fmt --alll` +3. `./run_checks.sh all` + +If you are getting test failures you can copy past the test name `foo_test` to run them directly with cargo until you get it working: + +``` +cargo test -p foo --lib --tests foo_test -- --nocapture + +``` + +where `foo` is the burn package (such as `burn-wgpu`) and `foo_test` is something like `fusion::base::tests::maxmin::tests::test_mean_dim_2d`. + +[^1]: for more on supertraits see [the advanced trait section of the rust book](https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#using-supertraits-to-require-one-traits-functionality-within-another-trait) +[^2]: wiki link for [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) +[^3]: for more information on unit structs see [the defining and instantiating structs section of the rust book](https://doc.rust-lang.org/book/ch05-01-defining-structs.html#unit-like-structs-without-any-fields) diff --git a/contributor-book/src/how-to-read-this-book.md b/contributor-book/src/how-to-read-this-book.md new file mode 100644 index 0000000000..9a5c4a7c7f --- /dev/null +++ b/contributor-book/src/how-to-read-this-book.md @@ -0,0 +1,14 @@ +# How to read this book + +Throughout this book, we try to keep the following structure + +## Linking + +When referring to structures or functions within codebase, we provide permalinks to the lines in specific commits, and indicate them by the relative path of their parent file from the project root. For example this is a reference to the `Tensor` struct in [`burn-tensor/src/tensor/api/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tensor/api/base.rs#L23) + +When some reference information is useful but is beyond the scope of contributing to burn, we provide that information in a footnote. To build on the previous example, the `Tensor` mentioned is what's referred to as a newtype struct[^1]. + +Direct hyperlinks are for tools and resources that are not part of the burn project, but are useful for contributing to it. For example, when working on implementing an op for autodiff, it is useful to use [symbolab](https://www.symbolab.com/) to calculate the left and right partial derivatives. + +[^1]: for more information on newtype please refer to [the Advanced Types chapter of the Rust Book](https://doc.rust-lang.org/book/ch19-04-advanced-types.html#using-the-newtype-pattern-for-type-safety-and-abstraction) + diff --git a/contributor-book/src/overview.md b/contributor-book/src/overview.md new file mode 100644 index 0000000000..8133fd7775 --- /dev/null +++ b/contributor-book/src/overview.md @@ -0,0 +1,17 @@ +# Overview + +Welcome to The Burn Contributor's Book 👋 + +This book will help you get acquainted with the internals of the Burn deep learning framework and provide some guidance on how to contribute to the project. + + We have crafted some sections for you: + +- [Basic Workflow](./basic-workflow): Much like the Burn Book for users, we'll start with the fundamentals, guiding you through tasks like setting up the development environment, how to run tests, and what you should check prior to each commit. + +- [Project Architecture](./project-architecture): This section will give you a more in-depth look at the architecture of burn + +- [Guides](./guides): We'll provide some guides on how to do specific tasks, such as adding a new operation to Burn. + +- [Frequency Encountered Issues](./frequently-encountered-issues.md): If you are running into an issue that has you stumped, this is the section to check out prior to asking on the discord. It's a collection of errors encountered by contributors, what caused them, and how they were resolved. + +As this book is geared more towards contributors rather than users of burn, we'll assume you have a good understanding of software development, but will make efforts to explain anything outside of that scope, or at least provide links to resources that explain it better than we can. From d9a637fb420eaf979e9c265b5afa0868547de95a Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 26 Jan 2024 11:29:31 -0600 Subject: [PATCH 02/32] forgot to enable mathjax for mdbook --- contributor-book/book.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contributor-book/book.toml b/contributor-book/book.toml index 4cf5f1f7d8..a92c7c79c7 100644 --- a/contributor-book/book.toml +++ b/contributor-book/book.toml @@ -5,9 +5,12 @@ authors = [ "Louis Fortier-Dubois", "Dilshod Tadjibaev", "Guillaume Lagrange", - "Joshua Ferguson" + "Joshua Ferguson", ] language = "en" multilingual = false src = "src" title = "The Burn Developer Book 🔥" + +[output.html] +mathjax-support = true From 8fc11bb042d332d6dcf42790c2d6e73654b2679b Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 26 Jan 2024 14:15:33 -0600 Subject: [PATCH 03/32] copied the contents of architecture.md, and fleshing things out --- contributor-book/src/SUMMARY.md | 16 +++- .../setting-up-the-environment.md | 1 - .../src/basic-workflow/testing.md | 1 - .../src/frequently-encountered-issues.md | 1 - .../frequently-encountered-issues/Readme.md | 1 + .../issues-while-adding-ops.md | 2 +- .../src/getting-started/ReadMe.md | 1 + .../configuring-your-editor.md | 24 ++++++ .../setting-up-the-environment.md | 31 +++++++ .../src/getting-started/testing.md | 19 +++++ contributor-book/src/guides/ReadMe.md | 3 + .../guides/adding-a-new-operation-to-burn.md | 18 +--- contributor-book/src/overview.md | 4 +- .../src/project-architecture/ReadMe.md | 19 +++++ .../src/project-architecture/Tensor.md | 21 +++++ .../src/project-architecture/backend.md | 35 ++++++++ .../src/project-architecture/module.md | 73 +++++++++++++++++ .../src/project-architecture/serialization.md | 82 +++++++++++++++++++ .../src/project-architecture/tensor.md | 1 + 19 files changed, 326 insertions(+), 27 deletions(-) delete mode 100644 contributor-book/src/basic-workflow/setting-up-the-environment.md delete mode 100644 contributor-book/src/basic-workflow/testing.md delete mode 100644 contributor-book/src/frequently-encountered-issues.md create mode 100644 contributor-book/src/frequently-encountered-issues/Readme.md create mode 100644 contributor-book/src/getting-started/ReadMe.md create mode 100644 contributor-book/src/getting-started/configuring-your-editor.md create mode 100644 contributor-book/src/getting-started/setting-up-the-environment.md create mode 100644 contributor-book/src/getting-started/testing.md create mode 100644 contributor-book/src/guides/ReadMe.md create mode 100644 contributor-book/src/project-architecture/ReadMe.md create mode 100644 contributor-book/src/project-architecture/Tensor.md create mode 100644 contributor-book/src/project-architecture/backend.md create mode 100644 contributor-book/src/project-architecture/module.md create mode 100644 contributor-book/src/project-architecture/serialization.md create mode 100644 contributor-book/src/project-architecture/tensor.md diff --git a/contributor-book/src/SUMMARY.md b/contributor-book/src/SUMMARY.md index f71e6b4069..c4dd4f8e6d 100644 --- a/contributor-book/src/SUMMARY.md +++ b/contributor-book/src/SUMMARY.md @@ -1,6 +1,14 @@ - [Overview](./overview.md) - [How to Read This Book](./how-to-read-this-book.md) -- [setting up the environment](./basic-workflow/setting-up-the-environment.md) -- [testing](./basic-workflow/testing.md) -- [adding a new operation to burn](./guides/adding-a-new-operation-to-burn.md) -- [Frequently Encountered Issues](./frequently-encountered-issues.md) \ No newline at end of file +- [Getting Started](./getting-started/ReadMe.md) + - [setting up the environment](./getting-started/setting-up-the-environment.md) + - [configuring your editor(optional)](./getting-started/configuring-your-editor.md) + - [testing](./getting-started/testing.md) +- [Architecture Overview](./project-architecture/ReadMe.md) + - [Modules](./project-architecture/module.md) + - [Serialization](./project-architecture/serialization.md) + - [Tensor](./project-architecture/tensor.md) + - [Backend](./project-architecture/backend.md) +- [Guides for Contributors](./guides/ReadMe.md) + - [adding a new operation to burn](./guides/adding-a-new-operation-to-burn.md) +- [Issue related to adding operators](./frequently-encountered-issues/issues-while-adding-ops.md) \ No newline at end of file diff --git a/contributor-book/src/basic-workflow/setting-up-the-environment.md b/contributor-book/src/basic-workflow/setting-up-the-environment.md deleted file mode 100644 index 5a009c6e25..0000000000 --- a/contributor-book/src/basic-workflow/setting-up-the-environment.md +++ /dev/null @@ -1 +0,0 @@ -# setting up the environment diff --git a/contributor-book/src/basic-workflow/testing.md b/contributor-book/src/basic-workflow/testing.md deleted file mode 100644 index 5697da2de5..0000000000 --- a/contributor-book/src/basic-workflow/testing.md +++ /dev/null @@ -1 +0,0 @@ -# testing diff --git a/contributor-book/src/frequently-encountered-issues.md b/contributor-book/src/frequently-encountered-issues.md deleted file mode 100644 index ce63065686..0000000000 --- a/contributor-book/src/frequently-encountered-issues.md +++ /dev/null @@ -1 +0,0 @@ -# Frequently Encountered Issues diff --git a/contributor-book/src/frequently-encountered-issues/Readme.md b/contributor-book/src/frequently-encountered-issues/Readme.md new file mode 100644 index 0000000000..161f626832 --- /dev/null +++ b/contributor-book/src/frequently-encountered-issues/Readme.md @@ -0,0 +1 @@ +This is a collection of issues people have encountered and asked about on the discord, and is separate from the guides since it often involves a wall of text that is only relevant to a small subset of contributors. \ No newline at end of file diff --git a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md index 1a173d50b1..f911894985 100644 --- a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md +++ b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md @@ -10,7 +10,7 @@ Most issues were fairly straight forward, and I was able to play whack-a-bug unt tests::maxmin::tests::test_mean_dim_2d stdout ---- thread 'tests::maxmin::tests::test_mean_dim_2d' panicked at burn-wgpu/src/lib.rs:49:5: assertion `left == right` failed left: Data { value: [1.0, 4.0], shape: Shape { dims: [2, 1] } } right: Data { value: [0.99999994, 3.9999998], shape: Shape { dims: [2, 1] } } ``` -If you encounter this, swap out the `assert_eq!` in the failing test for `tensor1.assert_approx_eq` +If you encounter this, swap out the `assert_eq!` in the failing test for `tensor1.assert_approx_eq` with `3` as the second argument. ### Mismatched types and missing functions diff --git a/contributor-book/src/getting-started/ReadMe.md b/contributor-book/src/getting-started/ReadMe.md new file mode 100644 index 0000000000..6b37be3b6e --- /dev/null +++ b/contributor-book/src/getting-started/ReadMe.md @@ -0,0 +1 @@ +This section is for setting up the environment and how to do basic development tasks such as running tests and checking your code before committing. \ No newline at end of file diff --git a/contributor-book/src/getting-started/configuring-your-editor.md b/contributor-book/src/getting-started/configuring-your-editor.md new file mode 100644 index 0000000000..fe823a53df --- /dev/null +++ b/contributor-book/src/getting-started/configuring-your-editor.md @@ -0,0 +1,24 @@ +# Configuring your editor + +These are not required, and most of this isn't specific to Burn, but it's definitely helpful if you haven't already done it. + +## VSCode + +1. Install the following extensions: + +- [rust-lang.rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) +- [tamasfe.even-better-toml](https://marketplace.visualstudio.com/items?itemName=tamasfe.even-better-toml) +- [serayuzgur.crates](https://marketplace.visualstudio.com/items?itemName=serayuzgur.crates) +- [vadimcn.vscode-lldb](https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb) + +2. Open `Command Palette` with Ctrl+Shift+P or F1 and type `LLDB: Generate Launch Configurations from Cargo.toml` then select it, this will generate a file that should be saved as `.vscode/launch.json`. + +3. Now you can enable breakpoint on code through IDE and then start debugging the library/binary you want, such as the following example: + +
+ +
+ +4. If you're creating a new library or binary, keep in mind to repeat the step 2 to always keep a fresh list of targets. + +## Have another editor? Open a PR! \ No newline at end of file diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md new file mode 100644 index 0000000000..0759a456ed --- /dev/null +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -0,0 +1,31 @@ +# setting up the environment + +There are a couple of tools that need to be installed, and commands to be familiar with, depending on what part of the project you plan on contributing to. This section should be up to date with current project practices (as of 2024-01-26) + +## General + +there are a few commands you want to run prior to any commit for a non-draft PR: + +1. `cargo clippy --fix --allow-dirty`, this will run clippy and fix any issues it can, the allow dirty flag is required whenever you have uncommitted changes +2. `cargo fmt --all`, this will run rustfmt on all files in the project +3. `./run_checks.sh all`, this is a script located in the project root that builds and tests the project. It is required that this pass prior to merging a PR. Fair warning, running these tests can take a while[^2]. + +## Updating the burn semver version + +To bump for the next version, install `cargo-edit` if its not on your system, and use this command: + +```sh +cargo set-version --bump minor +``` + +## Contributing to the Burn (Developer) Book + +Both the Burn book and the burn developer book are built with mdbook. To install mdbook, run the following command[^1]: + +```bash +cargo install mdbook +``` + + +[^1]: You might also want to install [cargo-update](https://github.com/nabijaczleweli/cargo-update) to easily keep your tools up to date, though it is in no way required. +[^2]: if your system is running into issues with memory and you are on linux you may want to switch to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console(and log in), and either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. \ No newline at end of file diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md new file mode 100644 index 0000000000..83eade8555 --- /dev/null +++ b/contributor-book/src/getting-started/testing.md @@ -0,0 +1,19 @@ +# testing + +## Test for TensorOps + +the following examples use matrix multiplication operation + +Test for Tensor operations (as in given this input, expect it match or approximate this output) are defined only in [`burn-tensor/src/test/ops`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/ops/matmul.rs#L1) and not in the backends, with the exception of `burn-autodiff`. These test are added to the `testgen_all` macro rule in [`burn-tensor/src/test/mod.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/mod.rs#L59). This is then propogated to the existing backends without any additional work. + +### Test for Autodiff + +the following examples use the power operation + +Test for autodiff go under [burn-autodiff/src/tests/foo.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replacing foo for whatever makes sense for your op), and for tensor operations both the left and right side need to be verified. The easiest way to do this, is to + +1. use small tensors with simple values +2. pop open a terminal and launch `ipython` import `numpy` (or just use [google colab](https://colab.google/) if you don't have the packages installed and don't want to install them), and do the calculations by hand. +3. comparing the actual to expected output for lhs, rhs and regular operation + +generally, it seems preferable to use `actual_output_tensor.to_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. \ No newline at end of file diff --git a/contributor-book/src/guides/ReadMe.md b/contributor-book/src/guides/ReadMe.md new file mode 100644 index 0000000000..07643c0595 --- /dev/null +++ b/contributor-book/src/guides/ReadMe.md @@ -0,0 +1,3 @@ +# Guides for Contributors + +The following guides are meant to help contributors trying to accomplish specific tasks, such as adding a new operations to Burn, generating test models for burn-import, or \ No newline at end of file diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index 259638e40c..c9e24b6d01 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -1,3 +1,4 @@ +# Adding a new operation to burn Let's discuss how one might go about adding new operators to burn, using the example of the recently added [pow operator](https://github.com/tracel-ai/burn/pull/1133/files) (as of 01/24/2024). In that PR, the following things took place (albeit not in this order) @@ -97,23 +98,6 @@ Let's review the changes made for pow starting from `src/burn` and moving to `sr And you're done! Congrats, you just fully added a new op to burn, and we are all one step closer to the answer to [are we learning yet?](https://www.arewelearningyet.com/) being "Yes, and it's freaking fast!". Buy yourself a coffee -## Making sure everything works - -This is definitely covered elsewhere but I'd run this prior to any commit - -1. `cargo clippy --fix --allow-dirty` -2. `cargo fmt --alll` -3. `./run_checks.sh all` - -If you are getting test failures you can copy past the test name `foo_test` to run them directly with cargo until you get it working: - -``` -cargo test -p foo --lib --tests foo_test -- --nocapture - -``` - -where `foo` is the burn package (such as `burn-wgpu`) and `foo_test` is something like `fusion::base::tests::maxmin::tests::test_mean_dim_2d`. - [^1]: for more on supertraits see [the advanced trait section of the rust book](https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#using-supertraits-to-require-one-traits-functionality-within-another-trait) [^2]: wiki link for [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) [^3]: for more information on unit structs see [the defining and instantiating structs section of the rust book](https://doc.rust-lang.org/book/ch05-01-defining-structs.html#unit-like-structs-without-any-fields) diff --git a/contributor-book/src/overview.md b/contributor-book/src/overview.md index 8133fd7775..f6e4f6ece2 100644 --- a/contributor-book/src/overview.md +++ b/contributor-book/src/overview.md @@ -6,12 +6,12 @@ This book will help you get acquainted with the internals of the Burn deep learn We have crafted some sections for you: -- [Basic Workflow](./basic-workflow): Much like the Burn Book for users, we'll start with the fundamentals, guiding you through tasks like setting up the development environment, how to run tests, and what you should check prior to each commit. +- [Getting Started](./getting-started): Much like the Burn Book for users, we'll start with the fundamentals, guiding you through tasks like setting up the development environment, how to run tests, and what you should check prior to each commit. - [Project Architecture](./project-architecture): This section will give you a more in-depth look at the architecture of burn - [Guides](./guides): We'll provide some guides on how to do specific tasks, such as adding a new operation to Burn. -- [Frequency Encountered Issues](./frequently-encountered-issues.md): If you are running into an issue that has you stumped, this is the section to check out prior to asking on the discord. It's a collection of errors encountered by contributors, what caused them, and how they were resolved. +- [Frequently Encountered Issues](./frequently-encountered-issues): If you are running into an issue that has you stumped, this is the section to check out prior to asking on the discord. It's a collection of errors encountered by contributors, what caused them, and how they were resolved. As this book is geared more towards contributors rather than users of burn, we'll assume you have a good understanding of software development, but will make efforts to explain anything outside of that scope, or at least provide links to resources that explain it better than we can. diff --git a/contributor-book/src/project-architecture/ReadMe.md b/contributor-book/src/project-architecture/ReadMe.md new file mode 100644 index 0000000000..f90d2bce79 --- /dev/null +++ b/contributor-book/src/project-architecture/ReadMe.md @@ -0,0 +1,19 @@ +# Project Architecture + +This Section documents most major architectural decisions with the reasoning behind them. + +**Sections** + +- [Module](./module.md) +- [Optimization](./module.md#optimization) + - [Constraints](./module.md#constraints) + - [Solution](./module.md#solution) +- [Serialization](./serialization.md) +- [Constraints](./serialization.md#constraints) +- [Solution](./serialization.md#solution) + - [Pros](./serialization.md#pros) + - [Cons](./serialization.md#cons) + - [Compatibility](./serialization.md#compatibility) +- [Tensor](./tensor.md) +- [Backend](./backend.md) + - [Autodiff](./backend.md#autodiff) diff --git a/contributor-book/src/project-architecture/Tensor.md b/contributor-book/src/project-architecture/Tensor.md new file mode 100644 index 0000000000..f1f2302683 --- /dev/null +++ b/contributor-book/src/project-architecture/Tensor.md @@ -0,0 +1,21 @@ +# Tensor + +A proper deep learning framework should have a fast tensor implementation with autodiff support, and Burn is no exception. +The tensor API abstracts away backend implementation details and focuses on usability without compromising performance. +To make it as easy as possible to use, there is only one tensor type, which is different from multiple tensor and deep learning crates in Rust. +Generic parameters are used instead to specialize the tensor type. + +- **B: Backend:** + The first argument is the backend on which the tensor implementation lies. +- **const D: usize:** + The second argument is the dimensionality of the tensor. +- **K: TensorKind:** + The third argument is the tensor kind, which can be either Float, Int or Bool. + By default, the tensor kind is set to Float, so for most tensors, the kind argument is not necessary. + +Having one struct for tensors reduces the complexity of the tensor API, which also means less duplicated documentation to write and maintain. + +Tensors are thread-safe, which means that you can send a tensor to another thread, and everything will work, including auto-differentiation. +Note that there are no in-place tensor operations since all tensor operations take owned tensors as parameters, which make it possible to mutate them. +Tensors can be shared simply by cloning them, but if there is only one reference to a tensor, the backend implementation is free to reuse the tensor's allocated data. +For more information about how it is done, you can have a look at this [blog post](https://burn.dev/blog/burn-rusty-approach-to-tensor-handling). \ No newline at end of file diff --git a/contributor-book/src/project-architecture/backend.md b/contributor-book/src/project-architecture/backend.md new file mode 100644 index 0000000000..0679f95cde --- /dev/null +++ b/contributor-book/src/project-architecture/backend.md @@ -0,0 +1,35 @@ + +# Backend + +The Backend trait abstracts multiple things: + +- Device type +- Float tensor type +- Bool tensor type +- Int tensor type +- Float element type +- Int element type +- Float tensor operations (kernels) +- Int tensor operations (kernels) +- Bool tensor operations (kernels) + +Even though having one type for tensors is convenient for the tensor API, it can be cumbersome when implementing a backend. +Therefore, backends can decide, through associated types, what types they want to use for their int, float, and bool tensors. +Since float and int can have multiple precisions, the float and int element types are also associated types that must be declared by the backend. + +Note that the backend chooses the precision and not the user. +Since not all backends will support the same element types, no assumptions must be made. +Therefore, there are no methods on tensors to change the precision, except for the `to_full_precision` function, which ensures numerical stability on the current backend. +Backend implementations can provide a way to choose the precision, which can be accomplished with a generic parameter (e.g. `NdArray`). + +To be as general as possible, tensor operations are implemented as plain functions. +There is no object or self, just functions that take tensors as input and often return tensors as output as well. +Backend implementations are free to use their own patterns to implement these kernels. +Note that Burn is a dynamic graph deep learning framework, so backends may have to implement asynchronous kernel executions for performance reasons. + +## Autodiff + +As of now, there is only one backend decorator that supports autodiff. +It follows the decorator pattern, making any backend differentiable. +However, the `AutodiffBackend` trait abstracts how gradients are calculated, and other approaches to autodiff might be added later. +For more information about how the current autodiff backend works, you can read this [blog post](https://burn.dev/blog/burn-rusty-approach-to-tensor-handling). diff --git a/contributor-book/src/project-architecture/module.md b/contributor-book/src/project-architecture/module.md new file mode 100644 index 0000000000..9b9d9896c5 --- /dev/null +++ b/contributor-book/src/project-architecture/module.md @@ -0,0 +1,73 @@ +# Module + +Modules are a way of creating neural network structures that can be easily optimized, saved, and loaded with little to no boilerplate. +Unlike other frameworks, a module does not force the declaration of the forward pass, leaving it up to the implementer to decide how it should be defined. + + +Additionally, most modules are created using a (de)serializable configuration, which defines the structure of the module and its hyper-parameters. +Parameters and hyper-parameters are not serialized into the same file and both are normally necessary to load a module for inference. + +## Optimization + +Optimization is normally done with gradient descent (or ascent for reinforcement learning), and it is important to provide an easy API for optimizing modules. + +### Constraints + +1. **Users should be able to control what is optimized.** + Modules can contain anything for maximum flexibility, but not everything needs to be optimized. +2. **Optimizers should have a serializable state that is updated during training.** + Many optimizers keep track of previous gradients to implement some form of momentum. + However, the state can be anything, not just tensors, allowing for easy implementation of any kind of optimizer. +3. **The learning rate can be updated during training.** + Learning rate schedulers are often used during training and should be considered as a key aspect. + +### Solution + +`Module` trait defined in [`burn-core/src/module/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/module/base.rs#L83) +`Optimizer` trait defined in [`burn-core/src/optim/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/base.rs#L8) + +The solution to this problem comprises multiple parts. +Firstly, the `Optimizer` trait is quite similar to the `Module` trait, in terms of saving and loading the state. Please refer to the [serialization](./serialization.md) section for more details. + +Secondly, two traits were created. +The `Optimizer` trait is general and relatively unopinionated, with a simple `step` method that takes a learning rate, a module, and the gradients. +The other trait, `SimpleOptimizer`, aims to provide an easier API for implementing new optimizers. +The goal is to allow implementations to avoid handling missing gradients, loading and exporting records, navigating the module parameter structure, handling tracked and untracked tensors, and other such tasks. + +Thirdly, each tensor that will be optimized needs to be wrapped into a `Param` struct, which gives them an ID used for (de)serialization and to associate the state of the optimizer to each parameter. +The `Module` trait has two ways to navigate over parameters. +The first one is the `map` function, which returns `Self` and makes it easy to implement any transformation and mutate all parameters. +The second one is the `visit` function, which has a similar signature but does not mutate the parameter tensors. + +#### SimpleOptimizer + +located in [`burn-core/src/optim/simple/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/simple/base.rs#L9) + +The `SimpleOptimizer` has two major assumptions: + +1. The state of the optimizer is linked to each parameter. + In other words, each parameter has its own optimizer state, decoupled from the other parameters. +2. The state of the optimizer implements `Record`, `Clone`, and has a `'static` lifetime. + +The benefits of those assumptions materialize in simplicity with little loss in flexibility. +The state associative type is also generic over the dimension, making it extremely easy to include tensors in the state that share the same dimensionality as its parameter. + +To wrap a simple optimizer into the more general `Optimizer` trait, the `OptimizerAdaptor` struct is used. + +#### OptimizerAdaptor + +Located in in [`burn-core/src/optim/simple/adapter.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/simple/adaptor.rs#L14) + +The `OptimizerAdaptor` is a simple struct composed of a `SimpleOptimizer` and a hashmap with all records associated with each parameter ID. + +When performing an optimization step, the adaptor handles the following: + +1. Updates each parameter tensor in the given module using the `Module::map` function. +2. Checks if a gradient for the current tensor exists. +3. Makes sure that the gradient, the tensor, and the optimizer state associated with the current parameter are on the same device. + The device can be different if the state is loaded from disk to restart training. +4. Performs the simple optimizer step using the inner tensor since the operations done by the optimizer should not be tracked in the autodiff graph. +5. Updates the state for the current parameter and returns the updated tensor, making sure it's properly registered into the autodiff graph if gradients are marked as required. + +Note that a parameter can still be updated by another process, as is the case with running metrics used in batch norm. +These tensors are still wrapped using the `Param` struct so that they are included in the module's state and given a proper parameter ID, but they are not registered in the autodiff graph. diff --git a/contributor-book/src/project-architecture/serialization.md b/contributor-book/src/project-architecture/serialization.md new file mode 100644 index 0000000000..58390fa737 --- /dev/null +++ b/contributor-book/src/project-architecture/serialization.md @@ -0,0 +1,82 @@ + +# Serialization + +An important aspect of a deep learning framework is the ability to save and load models from disk. +Despite appearing as a simple feature, it involves numerous constraints that require a proper solution. + +## Constraints + +1. **Users should be able to declare the precision of the model to be saved, independent of the backend in use.** + + The modules should not be duplicated in RAM in another precision to support this. + Conversion should be done lazily during (de)serialization. + +2. **Users should be able to add any field to a module, even fields that are not serializable.** + + This can include constants, database connections, other module references, or any other information. + Only parameters should be serialized since the structure of the module itself should be encapsulated with module configurations (hyper-parameters). + +3. **Users should be able to declare the format in which the module should be saved.** + + This can involve saving to a compressed JSON file or directly to bytes in memory for `no-std` environments. + +4. **Users should be able to create a module with its saved parameters without having to initialize the module first.** + + This will avoid unnecessary module initialization and tensor loading, resulting in reduced cold start when dealing with inference. + +In addition to all of these constraints, the solution should be easy to use. + +## Solution + +In order to be able to add any field to a module without requiring it to be (de)serializable, we decouple the module type from its state. +We create a new type for each module that only contains the parameters that need to be saved. +To generate that type automatically, the user must either declare which field is a parameter or a constant, or we assume that each field implements the module trait. + +The second solution was chosen as it simplifies the code generation and reduces the size of the user API. +This means that the `Module` trait should be implemented by [primitives types](./burn-core/src/module/param/primitive.rs). +The following diagrams highlight the main types and traits used in the solution. + +
+

Module Serialization Types

+ +
+ +The way the types interact with each other is pretty straightforward. +First, a module can be converted into a record using `into_record()`. +Note that tensors can be cloned, but it won't actually copy any data; it will create another reference to the same data. + +Then, a `Recorder` instance can be used to serialize any record. +The `Recorder` has the `PrecisionSettings` type as associate type, so any record will be serialized using the settings provided at the creation of the `Recorder` instance. +Note that tensors implement record, and their item is just a wrapper struct that contains information about the precision in which the tensor should be saved or loaded. +No actual copy of the tensor is made until this point. +The tensor is converted to the `Data` struct and then converted into the specified precision only when `serialize()` or `deserialize()` are called, which makes the whole process lazy. + +To recapitulate, the `Module` trait has an associated type that implements `Record`, which only contains the parameters of the model. +The `Record` trait has a generic associated type (GAT) that specifies a family of types that can be (de)serialized given any `PrecisionSettings`. +Records are therefore decoupled from the backend in use, and the saved items can be loaded on any backend with any precision, since the conversion is type-safe and done when `serialize()` and `deserialize()` are called. +All of the types are generated using simple derive macros without any conditional statements or complex syntax, as `Record` and `Module` are implemented for all primitive types. +This makes the code simple and easy to maintain. +In addition, you can extend the current system with your own `Recorder` and `PrecisionSettings` to control how your modules should be saved and loaded. + +### Pros + +- All constraints are respected. +- The code is simple and easy to maintain, with very few conditional statements. + It is just recursive data structures, where all the complexity is handled by the framework in primitive implementations. +- The user API is simple and small, with only two derives (`Record` and `Module`) and no additional attributes. +- Users can create their own `Module` and `Record` primitive types, which gives them the flexibility to control how their data is serialized without having to fork the framework. + +### Cons + +- There are more types, but most of them are automatically generated and single-purpose, so users don't need to interact with them for common use cases. + However, they can do so if necessary. +- When instantiating a new record manually, each field must be set to something, even if the type itself is `()`, which represents no value. + Since the code generation step uses associative types, it doesn't know that a field type is actually nothing. + Creating a record manually without using the generated function `into_record` or loading it from a file is only useful to load a set of parameters into a module from an arbitrary source. + Using the record may not be the optimal solution to this problem, and another API could be created in the future. + +### Compatibility + +Record may become incompatible with previous versions of Burn, depending on the chosen format. +The more compact format (bincode) store minimal information about the type, making it significantly smaller but less resilient to type changes such adding an optional field. +At some point, it might be necessary to provide a translation script that can translate a more resilient format from a previous version to a more compact one. \ No newline at end of file diff --git a/contributor-book/src/project-architecture/tensor.md b/contributor-book/src/project-architecture/tensor.md new file mode 100644 index 0000000000..820d98876f --- /dev/null +++ b/contributor-book/src/project-architecture/tensor.md @@ -0,0 +1 @@ +# Architecture: Tensor From c1b4daecc9dec8047e76c62b99483eb3c8383de8 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 26 Jan 2024 14:31:49 -0600 Subject: [PATCH 04/32] fixed issue with inline mathjax --- contributor-book/src/guides/adding-a-new-operation-to-burn.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index c9e24b6d01..f096e1324a 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -36,14 +36,14 @@ now for step 2. Since a significant number of the people reading this probably h In the case of pow, since this is a binary operation, the left and right functions are the partial derivatives with respect to the left and right sided tensors. -Let's define the operator as a function $f(x,y)=x^{y}$, where $x$ is the left hand tensor and $y$ is the right handed tensor. The two closures are defining the partial derivatives of $f$ with respect to $x$,$y$. The eli5 is treat the other variable as a constant +Let's define the operator as a function \\(f(x,y)=x^{y}\\) , where \\(x\\) is the left hand tensor and \\(y\\) is the right handed tensor. The two closures are defining the partial derivatives of \\(f\\) with respect to \\(x\\),\\(y\\). The eli5 is treat the other variable as a constant $$\frac{\delta }{\delta x} (x^{y})= y \cdot x^{y-1}$$ is the left handed closure, and $$\frac{\delta }{\delta y} (x^{y}) = x^{y} \cdot ln(x)$$ -is the right. If you aren't sure how to calculate these by hand, I recommend using [symbolab](https://www.symbolab.com/solver/partial-derivative-calculator/%5Cfrac%7B%5Cpartial%7D%7B%5Cpartial%20x%7D%5Cleft(x%5E%7By%7D%5Cright)?or=input), plug in your operator in terms of $x$ and $y$, and just swap out the variable $x$|$y$ in the partial derivative to get the other side. +is the right. If you aren't sure how to calculate these by hand, I recommend using [symbolab](https://www.symbolab.com/solver/partial-derivative-calculator/%5Cfrac%7B%5Cpartial%7D%7B%5Cpartial%20x%7D%5Cleft(x%5E%7By%7D%5Cright)?or=input), plug in your operator in terms of \\(x\\) and \\(y\\), and just swap out the variable \\(x\\)|\\(y\\) in the partial derivative to get the other side. ### Testing autodiff From e2738a87b79ff9c185bb9d1789315289d02c72a8 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 26 Jan 2024 15:35:43 -0600 Subject: [PATCH 05/32] fixed typos, added instructions --- .../src/frequently-encountered-issues/Readme.md | 4 +++- .../issues-while-adding-ops.md | 6 +++--- contributor-book/src/getting-started/ReadMe.md | 2 ++ .../src/getting-started/setting-up-the-environment.md | 1 + contributor-book/src/getting-started/testing.md | 2 +- contributor-book/src/guides/ReadMe.md | 2 +- contributor-book/src/project-architecture/module.md | 1 - 7 files changed, 11 insertions(+), 7 deletions(-) diff --git a/contributor-book/src/frequently-encountered-issues/Readme.md b/contributor-book/src/frequently-encountered-issues/Readme.md index 161f626832..65c3e341cd 100644 --- a/contributor-book/src/frequently-encountered-issues/Readme.md +++ b/contributor-book/src/frequently-encountered-issues/Readme.md @@ -1 +1,3 @@ -This is a collection of issues people have encountered and asked about on the discord, and is separate from the guides since it often involves a wall of text that is only relevant to a small subset of contributors. \ No newline at end of file +# Frequently Encountered Issues + +This is a collection of issues people have encountered and asked about on the discord, and is separate from the guides since it often involves a wall of text that is only relevant to a small subset of contributors. diff --git a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md index f911894985..89a1bdfa0b 100644 --- a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md +++ b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md @@ -1,8 +1,8 @@ -## Issues encountered while adding ops +# Issues encountered while adding ops Most issues were fairly straight forward, and I was able to play whack-a-bug until I hit ones were I needed outside help (on the discord channels), so I'll share some of the ones that had me stumped enough to ask for help in case you too encounter them. -### Off by .000001 errors +## Off by .000001 errors ```sh ---- fusion::base::tests::maxmin::tests::test_mean_dim_2d stdout ---- thread 'fusion::base::tests::maxmin::tests::test_mean_dim_2d' panicked at burn-wgpu/src/fusion/base.rs:185:5: assertion `left == right` failed left: Data { value: [1.0, 4.0], shape: Shape { dims: [2, 1] } } right: Data { value: [0.99999994, 3.9999998], shape: Shape { dims: [2, 1] } } ---- @@ -12,7 +12,7 @@ tests::maxmin::tests::test_mean_dim_2d stdout ---- thread 'tests::maxmin::tests: If you encounter this, swap out the `assert_eq!` in the failing test for `tensor1.assert_approx_eq` with `3` as the second argument. -### Mismatched types and missing functions +## Mismatched types and missing functions ```sh error[E0308]: mismatched types --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow.rs:48:45 | 48 | let pow1_out1 = input1.clone().powf(input1); | ---- ^^^^^^ expected `f32`, found `Tensor` | | | arguments to this method are incorrect | = note: expected type `f32` found struct `Tensor` diff --git a/contributor-book/src/getting-started/ReadMe.md b/contributor-book/src/getting-started/ReadMe.md index 6b37be3b6e..7e97955eb4 100644 --- a/contributor-book/src/getting-started/ReadMe.md +++ b/contributor-book/src/getting-started/ReadMe.md @@ -1 +1,3 @@ +# Getting Started + This section is for setting up the environment and how to do basic development tasks such as running tests and checking your code before committing. \ No newline at end of file diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index 0759a456ed..f295895829 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -26,6 +26,7 @@ Both the Burn book and the burn developer book are built with mdbook. To install cargo install mdbook ``` +also instead of running `./run_checks.sh all`, you can run `./run_checks.sh typo` to only check for misspellings. This will install [typo](https://crates.io/crates/typos-cli), and if any are encountered you should be able to run `typo -w /path/to/book` to fix them. [^1]: You might also want to install [cargo-update](https://github.com/nabijaczleweli/cargo-update) to easily keep your tools up to date, though it is in no way required. [^2]: if your system is running into issues with memory and you are on linux you may want to switch to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console(and log in), and either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. \ No newline at end of file diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index 83eade8555..5411063671 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -4,7 +4,7 @@ the following examples use matrix multiplication operation -Test for Tensor operations (as in given this input, expect it match or approximate this output) are defined only in [`burn-tensor/src/test/ops`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/ops/matmul.rs#L1) and not in the backends, with the exception of `burn-autodiff`. These test are added to the `testgen_all` macro rule in [`burn-tensor/src/test/mod.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/mod.rs#L59). This is then propogated to the existing backends without any additional work. +Test for Tensor operations (as in given this input, expect it match or approximate this output) are defined only in [`burn-tensor/src/test/ops`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/ops/matmul.rs#L1) and not in the backends, with the exception of `burn-autodiff`. These test are added to the `testgen_all` macro rule in [`burn-tensor/src/test/mod.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/mod.rs#L59). This is then propagated to the existing backends without any additional work. ### Test for Autodiff diff --git a/contributor-book/src/guides/ReadMe.md b/contributor-book/src/guides/ReadMe.md index 07643c0595..aad15f3626 100644 --- a/contributor-book/src/guides/ReadMe.md +++ b/contributor-book/src/guides/ReadMe.md @@ -1,3 +1,3 @@ # Guides for Contributors -The following guides are meant to help contributors trying to accomplish specific tasks, such as adding a new operations to Burn, generating test models for burn-import, or \ No newline at end of file +The following guides are meant to help contributors trying to accomplish specific tasks, such as adding a new operations to Burn or generating test models for burn-import. \ No newline at end of file diff --git a/contributor-book/src/project-architecture/module.md b/contributor-book/src/project-architecture/module.md index 9b9d9896c5..5a7d16e3db 100644 --- a/contributor-book/src/project-architecture/module.md +++ b/contributor-book/src/project-architecture/module.md @@ -3,7 +3,6 @@ Modules are a way of creating neural network structures that can be easily optimized, saved, and loaded with little to no boilerplate. Unlike other frameworks, a module does not force the declaration of the forward pass, leaving it up to the implementer to decide how it should be defined. - Additionally, most modules are created using a (de)serializable configuration, which defines the structure of the module and its hyper-parameters. Parameters and hyper-parameters are not serialized into the same file and both are normally necessary to load a module for inference. From 79c29c660f271b6b96d90251619ec83378a51457 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Sat, 27 Jan 2024 16:10:01 -0600 Subject: [PATCH 06/32] updating info on tensorops, fixing typos --- contributor-book/src/SUMMARY.md | 2 +- .../setting-up-the-environment.md | 15 +++++------- .../src/getting-started/testing.md | 5 ++-- .../guides/adding-a-new-operation-to-burn.md | 6 ++--- .../src/project-architecture/Tensor.md | 24 ++++++++++++++++++- .../src/project-architecture/tensor.md | 1 - 6 files changed, 36 insertions(+), 17 deletions(-) delete mode 100644 contributor-book/src/project-architecture/tensor.md diff --git a/contributor-book/src/SUMMARY.md b/contributor-book/src/SUMMARY.md index c4dd4f8e6d..3dd29bb913 100644 --- a/contributor-book/src/SUMMARY.md +++ b/contributor-book/src/SUMMARY.md @@ -7,7 +7,7 @@ - [Architecture Overview](./project-architecture/ReadMe.md) - [Modules](./project-architecture/module.md) - [Serialization](./project-architecture/serialization.md) - - [Tensor](./project-architecture/tensor.md) + - [Tensor](./project-architecture/Tensor.md) - [Backend](./project-architecture/backend.md) - [Guides for Contributors](./guides/ReadMe.md) - [adding a new operation to burn](./guides/adding-a-new-operation-to-burn.md) diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index f295895829..5cce94b442 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -4,7 +4,7 @@ There are a couple of tools that need to be installed, and commands to be famili ## General -there are a few commands you want to run prior to any commit for a non-draft PR: +There are a few commands you want to run prior to any commit for a non-draft PR: 1. `cargo clippy --fix --allow-dirty`, this will run clippy and fix any issues it can, the allow dirty flag is required whenever you have uncommitted changes 2. `cargo fmt --all`, this will run rustfmt on all files in the project @@ -12,21 +12,18 @@ there are a few commands you want to run prior to any commit for a non-draft PR: ## Updating the burn semver version -To bump for the next version, install `cargo-edit` if its not on your system, and use this command: +To bump for the next version, edit the semantic version number in `burn/Cargo.toml`, and then run `cargo update` to update the lock file. -```sh -cargo set-version --bump minor -``` -## Contributing to the Burn (Developer) Book +## Contributing to the Burn (Contributor) Book -Both the Burn book and the burn developer book are built with mdbook. To install mdbook, run the following command[^1]: +Both the Burn book and the burn contributor book are built with mdbook. To install mdbook, run the following command[^1]: ```bash cargo install mdbook ``` -also instead of running `./run_checks.sh all`, you can run `./run_checks.sh typo` to only check for misspellings. This will install [typo](https://crates.io/crates/typos-cli), and if any are encountered you should be able to run `typo -w /path/to/book` to fix them. +Also instead of running `./run_checks.sh all`, you can run `./run_checks.sh typo` to only check for misspellings. This will install [typo](https://crates.io/crates/typos-cli), and if any are encountered you should be able to run `typo -w /path/to/book` to fix them. [^1]: You might also want to install [cargo-update](https://github.com/nabijaczleweli/cargo-update) to easily keep your tools up to date, though it is in no way required. -[^2]: if your system is running into issues with memory and you are on linux you may want to switch to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console(and log in), and either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. \ No newline at end of file +[^2]: If your system is running into issues with memory and you are on linux you may want to switch to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console(and log in), and either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. \ No newline at end of file diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index 5411063671..e6416d89f1 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -10,10 +10,11 @@ Test for Tensor operations (as in given this input, expect it match or approxima the following examples use the power operation -Test for autodiff go under [burn-autodiff/src/tests/foo.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replacing foo for whatever makes sense for your op), and for tensor operations both the left and right side need to be verified. The easiest way to do this, is to +Test for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replacing foo for whatever makes sense for your op), and for tensor operations both the left and right side need to be verified. The easiest way to do this, is to 1. use small tensors with simple values 2. pop open a terminal and launch `ipython` import `numpy` (or just use [google colab](https://colab.google/) if you don't have the packages installed and don't want to install them), and do the calculations by hand. 3. comparing the actual to expected output for lhs, rhs and regular operation -generally, it seems preferable to use `actual_output_tensor.to_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. \ No newline at end of file +generally, it seems preferable to use `actual_output_tensor.to_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. + diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index f096e1324a..450ae49e2b 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -4,9 +4,9 @@ Let's discuss how one might go about adding new operators to burn, using the exa ## Adding the Op to burn-tensor -burn-tensor is the crate that defines all tensor operations that need to be implemented by the various backends. The core of this lies in `burn-tensor/src/tensor/api/numeric.rs`, which is home to the numeric trait and it's implementation for the different ensor types. The implementations for the `Foo` kind call the op defined under `tensor/src/tensor/op/foo_tensor.rs`Which defines ops specific to that type for the backend. +burn-tensor is the crate that defines all tensor operations that need to be implemented by the various backends. The core of this lies in `burn-tensor/src/tensor/api/numeric.rs`, which is home to the numeric trait and it's implementation for the different ensor types. The numeric trait is the home of all tensor operations that are numeric in nature that are shared by `Int` and `Float` Tensor types. More information on the relationship between Tensor modules can be found under the section for [Tensor Architecture](../project-architecture/Tensor.md#tensorops). -here is where pow was added to [burn-tensor/src/tensor/api/numeric.rs](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1618) +here is where pow was added to `burn-tensor/src/tensor/api/numeric.rs`: 1. for the [`Tensor` struct](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L553) 2. for the [numeric trait](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1618) @@ -20,7 +20,7 @@ The `Int` Tensor function use the ones defined for Float with 2 extra cast (LHS ### Adding Test -Additional Test should be added to `burn-tensor` under `burn-tensor/src/tests/ops/{op_name}.rs`(I would link to a file but I just realized I forgot to add one for non scalar pow ops). this file should then be added to `burn-tensor/src/tests/mod.rs` using the testgen macro. This test is then ran on every backend, save for those that require specific testing such as burn-autodiff +Additional Test should be added to `burn-tensor` under `burn-tensor/src/tests/ops/{op_name}.rs`. This file should then be added to `burn-tensor/src/tests/mod.rs` using the testgen macro. This test is then ran on every backend, save for those that require specific testing such as burn-autodiff ## Adding the Op to the burn-autodiff diff --git a/contributor-book/src/project-architecture/Tensor.md b/contributor-book/src/project-architecture/Tensor.md index f1f2302683..0a79248652 100644 --- a/contributor-book/src/project-architecture/Tensor.md +++ b/contributor-book/src/project-architecture/Tensor.md @@ -18,4 +18,26 @@ Having one struct for tensors reduces the complexity of the tensor API, which al Tensors are thread-safe, which means that you can send a tensor to another thread, and everything will work, including auto-differentiation. Note that there are no in-place tensor operations since all tensor operations take owned tensors as parameters, which make it possible to mutate them. Tensors can be shared simply by cloning them, but if there is only one reference to a tensor, the backend implementation is free to reuse the tensor's allocated data. -For more information about how it is done, you can have a look at this [blog post](https://burn.dev/blog/burn-rusty-approach-to-tensor-handling). \ No newline at end of file +For more information about how it is done, you can have a look at this [blog post](https://burn.dev/blog/burn-rusty-approach-to-tensor-handling). + +## TensorOps + +Operations on Tensors are defined in traits (generally part of the Backend Supertrait) and implemented for the Tensor struct. The appropriate parent trait of an op depends on the type of operation: + +- `base` => All tensor kinds should implement these operations (Reshape, into_data, etc.). The implementation is in `burn-tensor/src/tensor/api/base.rs`. +- `numeric` => All tensors that are numeric by nature should implement these operations (Add, Sub, Div, etc.). The implementation is in `burn-tensor/src/tensor/api/numeric.rs`. +- `Float` => Tensor operations are only available for float tensors. The implementation is in `burn-tensor/src/tensor/api/float.rs`. +- `Int` => Tensor operations are only available for int tensors. The implementation is in `burn-tensor/src/tensor/api/int.rs`. +- `bool` => Tensor operations are only available for bool tensors. The implementation is in `burn-tensor/src/tensor/api/bool.rs`. + +`Numeric` is directly implemented for `Float` and `Int` tensors, and in general, The implementations for these methods are calling the corresponding `{Int|Float}TensorOp` method defined in the backend supertrait. + +Anything that is implemented by numeric should have an implementation in the `{Int|Float}TensorOp` traits, though it may be avoidable if the operation for one type requires casting to the other type. To provide an example, Powf should be implemented for `Int` tensors, but it should not be an Int Tensor Operation. The LHS should be converted to a float, and the output should be converted back to an int. So it's possible to avoid implementing `IntTensorOp` altogether. + +additionally there are some operations that should be defined as functions instead of tensor/tensor op methods. these are: + +`module` => These should be exported as functions instead of methods on tensors. The implementation is in `burn-tensor/src/tensor/module.rs` (Might be moved to `tensor/api/module.rs`). +`activation` => These should also be exported as functions instead of methods on tensors. The implementation is in `burn-tensor/src/tensor/activation/base.rs` (Might be moved to `tensor/api/activation.rs`). + +Note that some activations are just a combination of backend operations and are not declared in `burn-tensor/src/tensor/ops/activation.rs` + diff --git a/contributor-book/src/project-architecture/tensor.md b/contributor-book/src/project-architecture/tensor.md deleted file mode 100644 index 820d98876f..0000000000 --- a/contributor-book/src/project-architecture/tensor.md +++ /dev/null @@ -1 +0,0 @@ -# Architecture: Tensor From cc93babe69696784c1fa2c944b60e68106ee4cc9 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Sat, 27 Jan 2024 17:59:59 -0600 Subject: [PATCH 07/32] added test for powf, fixed typos, updated index, etc --- burn-tensor/src/tests/mod.rs | 1 + burn-tensor/src/tests/ops/mod.rs | 1 + burn-tensor/src/tests/ops/powf.rs | 52 ++++++++++++ contributor-book/book.toml | 2 +- contributor-book/src/SUMMARY.md | 1 + .../src/getting-started/testing.md | 3 +- contributor-book/src/guides/README.md | 3 + contributor-book/src/guides/ReadMe.md | 2 - .../guides/adding-a-new-operation-to-burn.md | 4 +- .../guides/onnx-to-burn-conversion-tool.md | 80 +++++++++++++++++++ .../src/project-architecture/README.md | 19 +++++ .../src/project-architecture/ReadMe.md | 20 +---- 12 files changed, 162 insertions(+), 26 deletions(-) create mode 100644 burn-tensor/src/tests/ops/powf.rs create mode 100644 contributor-book/src/guides/README.md create mode 100644 contributor-book/src/guides/onnx-to-burn-conversion-tool.md create mode 100644 contributor-book/src/project-architecture/README.md diff --git a/burn-tensor/src/tests/mod.rs b/burn-tensor/src/tests/mod.rs index fbde19c8e5..4f654fa6f3 100644 --- a/burn-tensor/src/tests/mod.rs +++ b/burn-tensor/src/tests/mod.rs @@ -78,6 +78,7 @@ macro_rules! testgen_all { burn_tensor::testgen_tanh!(); burn_tensor::testgen_transpose!(); burn_tensor::testgen_tri!(); + burn_tensor::testgen_powf!(); // test stats burn_tensor::testgen_var!(); diff --git a/burn-tensor/src/tests/ops/mod.rs b/burn-tensor/src/tests/ops/mod.rs index 7dba1e2021..d2a954918e 100644 --- a/burn-tensor/src/tests/ops/mod.rs +++ b/burn-tensor/src/tests/ops/mod.rs @@ -28,6 +28,7 @@ mod mul; mod narrow; mod neg; mod one_hot; +mod powf; mod powf_scalar; mod random; mod recip; diff --git a/burn-tensor/src/tests/ops/powf.rs b/burn-tensor/src/tests/ops/powf.rs new file mode 100644 index 0000000000..127b7a6b83 --- /dev/null +++ b/burn-tensor/src/tests/ops/powf.rs @@ -0,0 +1,52 @@ +#[burn_tensor_testgen::testgen(powf)] +mod tests { + use super::*; + use burn_tensor::{Data, Tensor}; + + #[test] + fn should_support_powf_ops() { + let data = Data::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = Tensor::::from_data(data, &Default::default()); + let pow = Data::from([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor_pow = Tensor::::from_data(pow, &Default::default()); + let data_actual = tensor.powf(tensor_pow).into_data(); + let data_expected = Data::from([[1.0, 1.0, 4.0], [27.0, 256.0, 3125.0]]); + data_expected.assert_approx_eq(&data_actual, 3); + } + + #[test] + fn should_support_neg_power() { + let data = Data::from([[1.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + let tensor = Tensor::::from_data(data, &Default::default()); + let pow = Data::from([[-0.95, -0.67, -0.45], [-0.24, -0.5, -0.6]]); + let tensor_pow = Tensor::::from_data(pow, &Default::default()); + + let data_actual = tensor.powf(tensor_pow).into_data(); + + let data_expected = Data::from([[1., 1., 0.73204285], [0.76822936, 0.5, 0.38073079]]); + data_expected.assert_approx_eq(&data_actual, 3); + } + + #[test] + fn should_support_neg_values_with_even_power() { + let data = Data::from([[0.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]); + let tensor = Tensor::::from_data(data, &Default::default()); + let pow = Data::from([[2.0, 2.0, 4.0], [4.0, 4.0, 2.0]]); + let tensor_pow = Tensor::::from_data(pow, &Default::default()); + let data_actual = tensor.powf(tensor_pow).into_data(); + let data_expected = Data::from([[0.0, 1.0, 16.0], [81.0, 256.0, 25.0]]); + data_expected.assert_approx_eq(&data_actual, 3); + } + + #[test] + fn should_support_neg_values_with_odd_power() { + let data = Data::from([[0.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]); + let tensor = Tensor::::from_data(data, &Default::default()); + let pow = Data::from([[3.0, 3.0, 3.0], [3.0, 3.0, 3.0]]); + let tensor_pow = Tensor::::from_data(pow, &Default::default()); + let data_actual = tensor.powf(tensor_pow).into_data(); + + let data_expected = Data::from([[0.0, -1.0, -8.0], [-27.0, -64.0, -125.0]]); + data_expected.assert_approx_eq(&data_actual, 3); + } +} diff --git a/contributor-book/book.toml b/contributor-book/book.toml index a92c7c79c7..4f246b1e14 100644 --- a/contributor-book/book.toml +++ b/contributor-book/book.toml @@ -10,7 +10,7 @@ authors = [ language = "en" multilingual = false src = "src" -title = "The Burn Developer Book 🔥" +title = "The Burn Contributor Book 🔥" [output.html] mathjax-support = true diff --git a/contributor-book/src/SUMMARY.md b/contributor-book/src/SUMMARY.md index 3dd29bb913..28e8ea41af 100644 --- a/contributor-book/src/SUMMARY.md +++ b/contributor-book/src/SUMMARY.md @@ -10,5 +10,6 @@ - [Tensor](./project-architecture/Tensor.md) - [Backend](./project-architecture/backend.md) - [Guides for Contributors](./guides/ReadMe.md) + - [onnx to burn conversion tool: A Development Guide](./guides/onnx-to-burn-conversion-tool.md) - [adding a new operation to burn](./guides/adding-a-new-operation-to-burn.md) - [Issue related to adding operators](./frequently-encountered-issues/issues-while-adding-ops.md) \ No newline at end of file diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index e6416d89f1..1b2832d2ec 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -16,5 +16,4 @@ Test for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github 2. pop open a terminal and launch `ipython` import `numpy` (or just use [google colab](https://colab.google/) if you don't have the packages installed and don't want to install them), and do the calculations by hand. 3. comparing the actual to expected output for lhs, rhs and regular operation -generally, it seems preferable to use `actual_output_tensor.to_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. - +generally, it seems preferable to use `actual_output_tensor.into_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. diff --git a/contributor-book/src/guides/README.md b/contributor-book/src/guides/README.md new file mode 100644 index 0000000000..aad15f3626 --- /dev/null +++ b/contributor-book/src/guides/README.md @@ -0,0 +1,3 @@ +# Guides for Contributors + +The following guides are meant to help contributors trying to accomplish specific tasks, such as adding a new operations to Burn or generating test models for burn-import. \ No newline at end of file diff --git a/contributor-book/src/guides/ReadMe.md b/contributor-book/src/guides/ReadMe.md index aad15f3626..a6dec329dd 100644 --- a/contributor-book/src/guides/ReadMe.md +++ b/contributor-book/src/guides/ReadMe.md @@ -1,3 +1 @@ # Guides for Contributors - -The following guides are meant to help contributors trying to accomplish specific tasks, such as adding a new operations to Burn or generating test models for burn-import. \ No newline at end of file diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index 450ae49e2b..3420995ed6 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -20,7 +20,7 @@ The `Int` Tensor function use the ones defined for Float with 2 extra cast (LHS ### Adding Test -Additional Test should be added to `burn-tensor` under `burn-tensor/src/tests/ops/{op_name}.rs`. This file should then be added to `burn-tensor/src/tests/mod.rs` using the testgen macro. This test is then ran on every backend, save for those that require specific testing such as burn-autodiff +Additional Test should be added to `burn-tensor` under [`burn-tensor/src/tests/ops/{op_name}.rs`](https://github.com/tracel-ai/burn/burn-tensor/src/tests/ops/powf.rs), inserting the module name into ``burn-tensor/src/tests/ops/mod.rs`. Then add it to`testgen_all` macro under `burn-tensor/src/tests/mod.rs`. This test is automatically added to the backends so it isn't necessary to add them there, save for those that require specific testing such as`burn-autodiff` ## Adding the Op to the burn-autodiff @@ -86,7 +86,7 @@ There would have been a few other places additions would be necessary if some of ## Adding the Op to burn-import -I won't comment on generating the test onnx files or the test, as that is already covered [here](https://github.com/tracel-ai/burn/blob/main/burn-import/DEVELOPMENT.md#adding-new-operators), this is more about the specific changes you need to make when adding new operators after you have generated the tests. +I won't comment on generating the test onnx files or the test, as that is already covered [in the ONNX to burn guide](onnx-to-burn-conversion-tool.md#adding-new-operators), this is more about the specific changes you need to make when adding new operators after you have generated the tests. The crate is divided into two sections `src/burn` and `src/onnx`. The code under the former corresponds to the operation you've implemented earlier in this guide, and the latter to the operations defined in the onnx specification. So when you are loading a model, the operator is first parsed to an intermediate representation defined by `src/onnx`, and then mapped to a Burn operations defined under `src/burn/node`. diff --git a/contributor-book/src/guides/onnx-to-burn-conversion-tool.md b/contributor-book/src/guides/onnx-to-burn-conversion-tool.md new file mode 100644 index 0000000000..687428fe1b --- /dev/null +++ b/contributor-book/src/guides/onnx-to-burn-conversion-tool.md @@ -0,0 +1,80 @@ +# ONNX to Burn Conversion Tool: Development Guide + +This guide offers in-depth design insights and step-by-step procedures for developers working on the +ONNX to Burn conversion tool. This tool allows the importation of ONNX models into the Burn deep +learning framework written in Rust. It converts both ONNX models to Rust source code and model +weights to Burn state files. + +## Table of Contents + +1. [Design Overview](#Design-Overview) + 1. [Design Goals](#Design-Goals) + 2. [Design Decisions](#Design-Decisions) +2. [Adding New Operators](#Adding-New-Operators) +3. [Testing](#Testing) +4. [Resources](#Resources) + +## Design Overview + +### Design Goals + +- Perform best-effort conversion of ONNX models to Rust source code via Burn APIs. +- Convert ONNX model weights to Burn state files. +- Support ONNX models generated by PyTorch (ONNX Opset 16). +- Produce easy-to-understand and modifiable models. +- Ensure the generated models are trainable using Burn APIs. + +### Design Decisions + +- Limit interaction with ONNX to the Intermediate Representation (IR) stage to simplify the process. +- Ensure operator behavior consistency across different OpSet versions. +- Exclude any ONNX/Protobuf-specific logic from the Burn graph. + +The conversion process involves three main stages: + +1. Convert ONNX model to Intermediate Representation (IR). +2. Translate IR to a Burn graph. +3. Generate Rust source code from the Burn graph. + +## Adding New Operators + +To extend `burn-import` with support for new ONNX operators, follow these steps: + +1. **Create PyTorch Script**: Place a PyTorch script using the new operator under + `./burn-import/onnx-tests/tests//.py`. Make sure to print both input and output tensors + for end-to-end testing. + +2. **Generate ONNX Model**: Run the PyTorch script to produce an ONNX model. + +3. **Visualize ONNX Model**: Use [Netron](https://github.com/lutzroeder/netron) to verify the ONNX + model contains the expected operators. + +4. **Generate IR and Burn Graph**: Navigate to `./burn-import/` and run: + + ``` + cargo r -- ./onnx-tests/tests//.onnx ./out + ``` + +5. **Implement Missing Operators**: If you encounter an error stating that an operator is + unsupported, implement it. The `./out/my-model.graph.txt` should provide relevant information. + +6. **Inspect Generated Files**: The `my-model.graph.txt` contains IR details, `my-model.rs` holds + the Burn model in Rust code, and `my-model.json` includes the model data. + +7. **Add End-to-End Test**: Include the test in `./burn-import/onnx-tests/tests/onnx_tests.rs`. + Further details can be found in the [onnx-tests README](./burn-import/onnx-tests/README.md). + +## Testing + +- Unit tests for the Burn graph to Rust source code conversion are mandatory. +- End-to-end tests should include a test ONNX model and its expected output for each operator. + +## Resources + +1. [PyTorch to ONNX](https://pytorch.org/docs/stable/onnx.html) +2. [ONNX to PyTorch](https://github.com/ENOT-AutoDL/onnx2torch) +3. [ONNX Introduction](https://onnx.ai/onnx/intro/) +4. [ONNX Operators](https://onnx.ai/onnx/operators/index.html) +5. [ONNX Protos](https://onnx.ai/onnx/api/classes.html) +6. [ONNX Optimizer](https://github.com/onnx/optimizer) +7. [Netron](https://github.com/lutzroeder/netron) diff --git a/contributor-book/src/project-architecture/README.md b/contributor-book/src/project-architecture/README.md new file mode 100644 index 0000000000..f90d2bce79 --- /dev/null +++ b/contributor-book/src/project-architecture/README.md @@ -0,0 +1,19 @@ +# Project Architecture + +This Section documents most major architectural decisions with the reasoning behind them. + +**Sections** + +- [Module](./module.md) +- [Optimization](./module.md#optimization) + - [Constraints](./module.md#constraints) + - [Solution](./module.md#solution) +- [Serialization](./serialization.md) +- [Constraints](./serialization.md#constraints) +- [Solution](./serialization.md#solution) + - [Pros](./serialization.md#pros) + - [Cons](./serialization.md#cons) + - [Compatibility](./serialization.md#compatibility) +- [Tensor](./tensor.md) +- [Backend](./backend.md) + - [Autodiff](./backend.md#autodiff) diff --git a/contributor-book/src/project-architecture/ReadMe.md b/contributor-book/src/project-architecture/ReadMe.md index f90d2bce79..42e34a2c8e 100644 --- a/contributor-book/src/project-architecture/ReadMe.md +++ b/contributor-book/src/project-architecture/ReadMe.md @@ -1,19 +1 @@ -# Project Architecture - -This Section documents most major architectural decisions with the reasoning behind them. - -**Sections** - -- [Module](./module.md) -- [Optimization](./module.md#optimization) - - [Constraints](./module.md#constraints) - - [Solution](./module.md#solution) -- [Serialization](./serialization.md) -- [Constraints](./serialization.md#constraints) -- [Solution](./serialization.md#solution) - - [Pros](./serialization.md#pros) - - [Cons](./serialization.md#cons) - - [Compatibility](./serialization.md#compatibility) -- [Tensor](./tensor.md) -- [Backend](./backend.md) - - [Autodiff](./backend.md#autodiff) +# Architecture Overview From fb64d114415c7b3854171a16a26fcfc4b10e89c8 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Sun, 28 Jan 2024 14:15:20 -0600 Subject: [PATCH 08/32] fixed issue with burn-wgpu 0^0 --- burn-wgpu/src/codegen/function.rs | 79 ++++++++++++++++++++++++++----- burn-wgpu/src/codegen/kernel.rs | 19 +++++--- contributor-book/src/SUMMARY.md | 11 +++-- 3 files changed, 86 insertions(+), 23 deletions(-) diff --git a/burn-wgpu/src/codegen/function.rs b/burn-wgpu/src/codegen/function.rs index 60747b70c0..5e1b5e8c3a 100644 --- a/burn-wgpu/src/codegen/function.rs +++ b/burn-wgpu/src/codegen/function.rs @@ -5,6 +5,7 @@ use std::fmt::Display; /// Not all functions are native to WGSL, so this struct allows to support more functions. #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] pub enum Function { + PowfScalar(Item), Powf(Item), Erf(Item), #[cfg(target_os = "macos")] @@ -14,6 +15,7 @@ pub enum Function { impl Display for Function { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Function::PowfScalar(elem) => format_powf_scalar(f, elem), Function::Powf(elem) => format_powf(f, elem), Function::Erf(elem) => format_erf(f, elem), #[cfg(target_os = "macos")] @@ -22,14 +24,62 @@ impl Display for Function { } } -fn format_powf(f: &mut core::fmt::Formatter<'_>, item: &Item) -> core::fmt::Result { - let elem = item.elem(); +fn format_powf_scalar(f: &mut core::fmt::Formatter<'_>, item: &Item) -> core::fmt::Result { + base_powf_fmt(f, item)?; + + match item { + Item::Vec4(elem) => f.write_fmt(format_args!( + " +fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ + return vec4( + powf_scalar(lhs[0], rhs), + powf_scalar(lhs[1], rhs), + powf_scalar(lhs[2], rhs), + powf_scalar(lhs[3], rhs), + ); +}} +" + )), + Item::Vec3(elem) => f.write_fmt(format_args!( + " +fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ + return vec3( + powf_scalar(lhs[0], rhs), + powf_scalar(lhs[1], rhs), + powf_scalar(lhs[2], rhs), + ); +}} +" + )), + Item::Vec2(elem) => f.write_fmt(format_args!( + " +fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ + return vec2( + powf_scalar(lhs[0], rhs), + powf_scalar(lhs[1], rhs), + ); +}} +" + )), + Item::Scalar(elem) => f.write_fmt(format_args!( + " +fn powf(lhs: {elem}, rhs: {elem}) -> {elem} {{ + return powf_scalar(lhs, rhs); +}} +" + )), + } +} +fn base_powf_fmt(f: &mut std::fmt::Formatter<'_>, item: &Item) -> Result<(), std::fmt::Error> { + let elem = item.elem(); f.write_fmt(format_args!( " fn powf_scalar(lhs: {elem}, rhs: {elem}) -> {elem} {{ let modulo = rhs % 2.0; - + if rhs==0.0 {{ + return 1.0; + }} if (modulo == 0.0) {{ // Even number return pow(abs(lhs), rhs); @@ -43,16 +93,21 @@ fn powf_scalar(lhs: {elem}, rhs: {elem}) -> {elem} {{ }} " ))?; + Ok(()) +} + +fn format_powf(f: &mut core::fmt::Formatter<'_>, item: &Item) -> core::fmt::Result { + base_powf_fmt(f, item)?; match item { Item::Vec4(elem) => f.write_fmt(format_args!( " fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ return vec4( - powf_scalar(lhs[0], rhs), - powf_scalar(lhs[1], rhs), - powf_scalar(lhs[2], rhs), - powf_scalar(lhs[3], rhs), + powf_scalar(lhs[0], rhs[0]), + powf_scalar(lhs[1], rhs[1]), + powf_scalar(lhs[2], rhs[2]), + powf_scalar(lhs[3], rhs[3]), ); }} " @@ -61,9 +116,9 @@ fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ " fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ return vec3( - powf_scalar(lhs[0], rhs), - powf_scalar(lhs[1], rhs), - powf_scalar(lhs[2], rhs), + powf_scalar(lhs[0], rhs[0]), + powf_scalar(lhs[1], rhs[1]), + powf_scalar(lhs[2], rhs[2]), ); }} " @@ -72,8 +127,8 @@ fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ " fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ return vec2( - powf_scalar(lhs[0], rhs), - powf_scalar(lhs[1], rhs), + powf_scalar(lhs[0], rhs[0]), + powf_scalar(lhs[1], rhs[1]), ); }} " diff --git a/burn-wgpu/src/codegen/kernel.rs b/burn-wgpu/src/codegen/kernel.rs index 1bc928cced..0d0ff1c7f2 100644 --- a/burn-wgpu/src/codegen/kernel.rs +++ b/burn-wgpu/src/codegen/kernel.rs @@ -202,13 +202,20 @@ impl ElemWiseKernelCodegen { match ops { Operator::Powf { lhs: _, - rhs: _, + rhs, out: _, - } => { - register_function(Function::Powf( - Item::Scalar(Elem::F32).vectorize(self.vectorization), - )); - } + } => match rhs { + Variable::Scalar(_, _) => { + register_function(Function::PowfScalar( + Item::Scalar(Elem::F32).vectorize(self.vectorization), + )); + } + _ => { + register_function(Function::Powf( + Item::Scalar(Elem::F32).vectorize(self.vectorization), + )); + } + }, Operator::Erf { input: _, out: _ } => { register_function(Function::Erf( Item::Scalar(Elem::F32).vectorize(self.vectorization), diff --git a/contributor-book/src/SUMMARY.md b/contributor-book/src/SUMMARY.md index 28e8ea41af..38c205e2e3 100644 --- a/contributor-book/src/SUMMARY.md +++ b/contributor-book/src/SUMMARY.md @@ -1,8 +1,8 @@ - [Overview](./overview.md) - [How to Read This Book](./how-to-read-this-book.md) - [Getting Started](./getting-started/ReadMe.md) - - [setting up the environment](./getting-started/setting-up-the-environment.md) - - [configuring your editor(optional)](./getting-started/configuring-your-editor.md) + - [Setting Up The Environment](./getting-started/setting-up-the-environment.md) + - [Configuring Your Editor(Optional)](./getting-started/configuring-your-editor.md) - [testing](./getting-started/testing.md) - [Architecture Overview](./project-architecture/ReadMe.md) - [Modules](./project-architecture/module.md) @@ -10,6 +10,7 @@ - [Tensor](./project-architecture/Tensor.md) - [Backend](./project-architecture/backend.md) - [Guides for Contributors](./guides/ReadMe.md) - - [onnx to burn conversion tool: A Development Guide](./guides/onnx-to-burn-conversion-tool.md) - - [adding a new operation to burn](./guides/adding-a-new-operation-to-burn.md) -- [Issue related to adding operators](./frequently-encountered-issues/issues-while-adding-ops.md) \ No newline at end of file + - [Onnx To Burn Conversion Tool: A Development Guide](./guides/onnx-to-burn-conversion-tool.md) + - [Adding a New Operation to Burn](./guides/adding-a-new-operation-to-burn.md) + - [Frequently Encountered Issues](./guides/frequently-encountered-issues/ReadMe.md) + - [Issues Related To Adding Operators](./guides/frequently-encountered-issues/issues-while-adding-ops.md) \ No newline at end of file From bcc232e03f863b2132cb70b78a53f5a53045f3de Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Sun, 28 Jan 2024 14:15:57 -0600 Subject: [PATCH 09/32] formatting --- burn-wgpu/src/codegen/function.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/burn-wgpu/src/codegen/function.rs b/burn-wgpu/src/codegen/function.rs index 5e1b5e8c3a..c4bb5c5e88 100644 --- a/burn-wgpu/src/codegen/function.rs +++ b/burn-wgpu/src/codegen/function.rs @@ -77,7 +77,7 @@ fn base_powf_fmt(f: &mut std::fmt::Formatter<'_>, item: &Item) -> Result<(), std " fn powf_scalar(lhs: {elem}, rhs: {elem}) -> {elem} {{ let modulo = rhs % 2.0; - if rhs==0.0 {{ + if rhs == 0.0 {{ return 1.0; }} if (modulo == 0.0) {{ From 227a5ea03c8d0bca4e69e2cdc8b2ddce5c76f223 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Sun, 28 Jan 2024 14:37:34 -0600 Subject: [PATCH 10/32] updated section for adding ops to wgpu --- contributor-book/src/guides/adding-a-new-operation-to-burn.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index 3420995ed6..6aa00a31df 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -81,8 +81,10 @@ here is where code was added 1. to the implementation of [`TensorOps` under `burn-wgpu/src/ops/float_ops.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/float_ops.rs#L513) 2. the function being called was added to [burn-wgpu/src/ops/numeric.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/numeric.rs#L199) +3. the call to the fmt function use to generate wgsl code in [`burn-wgpu/src/codegen/kernel.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/codegen/kernel.rs#L208) +4. A custom function generator was added to [`burn-wgpu/src/codegen/function.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/codegen/function.rs#L99) -There would have been a few other places additions would be necessary if some of the code for powf hadn't already been defined, such as [`burn-wgpu/src/fusion/elemwise/builder.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/fusion/elemwise/optimization.rs) and [`burn-wgpu/src/fusion/elemwise/builder.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/fusion/elemwise/builder.rs#L276), but I don't know if there are other places within `burn-wgpu` where additions would be necessary. +Much of the logic for powf had already been defined, so not much needed to be added. The reason for the custom powf function is generated, rather than directly using the underlying function as is the case with other operations, is due to issues with case handling of the wgsl pow function, like 0 to the 0 power being 1, and any negative number to an even power being positive. We reused as much as the existing logic as possible, such as the operation output description generation in [`burn-wgpu/src/fusion/elemwise/builder.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/fusion/elemwise/optimization.rs) and then branched at the last point based off the var type of the rhs. I don't know if there are other places within `burn-wgpu` where additions would have been necessary otherwise. ## Adding the Op to burn-import From 9ea4498c04e9211c6ac2ab970f8ce223b2b806f4 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:11:07 -0600 Subject: [PATCH 11/32] changed powf_scalar to powf_primitive for wgpu code --- burn-wgpu/src/codegen/function.rs | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/burn-wgpu/src/codegen/function.rs b/burn-wgpu/src/codegen/function.rs index c4bb5c5e88..f2ecffd6b0 100644 --- a/burn-wgpu/src/codegen/function.rs +++ b/burn-wgpu/src/codegen/function.rs @@ -32,10 +32,10 @@ fn format_powf_scalar(f: &mut core::fmt::Formatter<'_>, item: &Item) -> core::fm " fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ return vec4( - powf_scalar(lhs[0], rhs), - powf_scalar(lhs[1], rhs), - powf_scalar(lhs[2], rhs), - powf_scalar(lhs[3], rhs), + powf_primitive(lhs[0], rhs), + powf_primitive(lhs[1], rhs), + powf_primitive(lhs[2], rhs), + powf_primitive(lhs[3], rhs), ); }} " @@ -44,9 +44,9 @@ fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ " fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ return vec3( - powf_scalar(lhs[0], rhs), - powf_scalar(lhs[1], rhs), - powf_scalar(lhs[2], rhs), + powf_primitive(lhs[0], rhs), + powf_primitive(lhs[1], rhs), + powf_primitive(lhs[2], rhs), ); }} " @@ -55,8 +55,8 @@ fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ " fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ return vec2( - powf_scalar(lhs[0], rhs), - powf_scalar(lhs[1], rhs), + powf_primitive(lhs[0], rhs), + powf_primitive(lhs[1], rhs), ); }} " @@ -64,7 +64,7 @@ fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ Item::Scalar(elem) => f.write_fmt(format_args!( " fn powf(lhs: {elem}, rhs: {elem}) -> {elem} {{ - return powf_scalar(lhs, rhs); + return powf_primitive(lhs, rhs); }} " )), @@ -75,7 +75,7 @@ fn base_powf_fmt(f: &mut std::fmt::Formatter<'_>, item: &Item) -> Result<(), std let elem = item.elem(); f.write_fmt(format_args!( " -fn powf_scalar(lhs: {elem}, rhs: {elem}) -> {elem} {{ +fn powf_primitive(lhs: {elem}, rhs: {elem}) -> {elem} {{ let modulo = rhs % 2.0; if rhs == 0.0 {{ return 1.0; @@ -104,10 +104,10 @@ fn format_powf(f: &mut core::fmt::Formatter<'_>, item: &Item) -> core::fmt::Resu " fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ return vec4( - powf_scalar(lhs[0], rhs[0]), - powf_scalar(lhs[1], rhs[1]), - powf_scalar(lhs[2], rhs[2]), - powf_scalar(lhs[3], rhs[3]), + powf_primitive(lhs[0], rhs[0]), + powf_primitive(lhs[1], rhs[1]), + powf_primitive(lhs[2], rhs[2]), + powf_primitive(lhs[3], rhs[3]), ); }} " @@ -116,9 +116,9 @@ fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ " fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ return vec3( - powf_scalar(lhs[0], rhs[0]), - powf_scalar(lhs[1], rhs[1]), - powf_scalar(lhs[2], rhs[2]), + powf_primitive(lhs[0], rhs[0]), + powf_primitive(lhs[1], rhs[1]), + powf_primitive(lhs[2], rhs[2]), ); }} " @@ -127,8 +127,8 @@ fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ " fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ return vec2( - powf_scalar(lhs[0], rhs[0]), - powf_scalar(lhs[1], rhs[1]), + powf_primitive(lhs[0], rhs[0]), + powf_primitive(lhs[1], rhs[1]), ); }} " @@ -136,7 +136,7 @@ fn powf(lhs: {item}, rhs: {elem}) -> {item} {{ Item::Scalar(elem) => f.write_fmt(format_args!( " fn powf(lhs: {elem}, rhs: {elem}) -> {elem} {{ - return powf_scalar(lhs, rhs); + return powf_primitive(lhs, rhs); }} " )), From 15c0212aaf8964c70395a013fc8463f04bd21220 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:13:59 -0600 Subject: [PATCH 12/32] Update contributor-book/src/getting-started/testing.md Co-authored-by: Sylvain Benner --- contributor-book/src/getting-started/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index 1b2832d2ec..d2dbb8b33c 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -13,7 +13,7 @@ the following examples use the power operation Test for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replacing foo for whatever makes sense for your op), and for tensor operations both the left and right side need to be verified. The easiest way to do this, is to 1. use small tensors with simple values -2. pop open a terminal and launch `ipython` import `numpy` (or just use [google colab](https://colab.google/) if you don't have the packages installed and don't want to install them), and do the calculations by hand. +2. pop open a terminal, launch `ipython` and import `numpy` then do the calculations by hand. You can also use [google colab](https://colab.google/) if you prefer so that you don't have to install the packages on your system. 3. comparing the actual to expected output for lhs, rhs and regular operation generally, it seems preferable to use `actual_output_tensor.into_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. From f408efff8ef9934378098e84482455241dcfe46a Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:14:10 -0600 Subject: [PATCH 13/32] Update contributor-book/src/guides/adding-a-new-operation-to-burn.md Co-authored-by: Sylvain Benner --- contributor-book/src/guides/adding-a-new-operation-to-burn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index 6aa00a31df..e80c4bcb7d 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -1,6 +1,6 @@ # Adding a new operation to burn -Let's discuss how one might go about adding new operators to burn, using the example of the recently added [pow operator](https://github.com/tracel-ai/burn/pull/1133/files) (as of 01/24/2024). In that PR, the following things took place (albeit not in this order) +Let's discuss how one might go about adding new operators to Burn, using the example of the recently added [pow operator](https://github.com/tracel-ai/burn/pull/1133/files) (as of 01/24/2024). In that PR, the following things took place (albeit not in this order) ## Adding the Op to burn-tensor From b4f84d09d95cceb65f357cd68a6219b8d2a9033a Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:14:21 -0600 Subject: [PATCH 14/32] Update contributor-book/src/guides/adding-a-new-operation-to-burn.md Co-authored-by: Sylvain Benner --- contributor-book/src/guides/adding-a-new-operation-to-burn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index e80c4bcb7d..8b2c2bb8ec 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -4,7 +4,7 @@ Let's discuss how one might go about adding new operators to Burn, using the exa ## Adding the Op to burn-tensor -burn-tensor is the crate that defines all tensor operations that need to be implemented by the various backends. The core of this lies in `burn-tensor/src/tensor/api/numeric.rs`, which is home to the numeric trait and it's implementation for the different ensor types. The numeric trait is the home of all tensor operations that are numeric in nature that are shared by `Int` and `Float` Tensor types. More information on the relationship between Tensor modules can be found under the section for [Tensor Architecture](../project-architecture/Tensor.md#tensorops). +`burn-tensor` is the crate that defines all tensor operations that need to be implemented by the various backends. The core of this lies in `burn-tensor/src/tensor/api/numeric.rs`, which is home to the numeric trait and its implementation for the different tensor types. The numeric trait is the home of all tensor operations that are numeric in nature and that are shared by `Int` and `Float` Tensor types. More information on the relationship between Tensor modules can be found under the section for [Tensor Architecture](../project-architecture/Tensor.md#tensorops). here is where pow was added to `burn-tensor/src/tensor/api/numeric.rs`: From 151f444f47ebf07a5c6cdd17141d176385ff942a Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:14:30 -0600 Subject: [PATCH 15/32] Update contributor-book/src/getting-started/testing.md Co-authored-by: Sylvain Benner --- contributor-book/src/getting-started/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index d2dbb8b33c..b2cc8c3cd9 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -14,6 +14,6 @@ Test for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github 1. use small tensors with simple values 2. pop open a terminal, launch `ipython` and import `numpy` then do the calculations by hand. You can also use [google colab](https://colab.google/) if you prefer so that you don't have to install the packages on your system. -3. comparing the actual to expected output for lhs, rhs and regular operation +3. compare the actual output to the expected output for lhs, rhs and regular operation generally, it seems preferable to use `actual_output_tensor.into_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. From 8c67766e51dbd79b20ed24fc1659ec27796a9b02 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:14:38 -0600 Subject: [PATCH 16/32] Update contributor-book/src/getting-started/testing.md Co-authored-by: Sylvain Benner --- contributor-book/src/getting-started/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index b2cc8c3cd9..540e531a10 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -10,7 +10,7 @@ Test for Tensor operations (as in given this input, expect it match or approxima the following examples use the power operation -Test for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replacing foo for whatever makes sense for your op), and for tensor operations both the left and right side need to be verified. The easiest way to do this, is to +Tests for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replace `foo` for whatever makes sense for your op), and for tensor operations both the left and right sides need to be verified. The easiest way to do this, is to: 1. use small tensors with simple values 2. pop open a terminal, launch `ipython` and import `numpy` then do the calculations by hand. You can also use [google colab](https://colab.google/) if you prefer so that you don't have to install the packages on your system. From b3126073e904925bd2b48b91fe52a65b186bae65 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:17:33 -0600 Subject: [PATCH 17/32] Update contributor-book/src/getting-started/setting-up-the-environment.md Co-authored-by: Sylvain Benner --- .../src/getting-started/setting-up-the-environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index 5cce94b442..a3418f6451 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -8,7 +8,7 @@ There are a few commands you want to run prior to any commit for a non-draft PR: 1. `cargo clippy --fix --allow-dirty`, this will run clippy and fix any issues it can, the allow dirty flag is required whenever you have uncommitted changes 2. `cargo fmt --all`, this will run rustfmt on all files in the project -3. `./run_checks.sh all`, this is a script located in the project root that builds and tests the project. It is required that this pass prior to merging a PR. Fair warning, running these tests can take a while[^2]. +3. `./run_checks.sh all`, this is a script located in the project root that builds and tests the project. It is required that this passes prior to merging a PR. Fair warning, running these tests can take a while[^2]. ## Updating the burn semver version From 5ac299a049fd39707e7c6644069552e8fa9bc6f1 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:29:28 -0600 Subject: [PATCH 18/32] implementeds suggestions by @syl20bnr --- contributor-book/book.toml | 1 + .../frequently-encountered-issues/issues-while-adding-ops.md | 4 ++-- .../src/getting-started/setting-up-the-environment.md | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/contributor-book/book.toml b/contributor-book/book.toml index 4f246b1e14..0be079c0d5 100644 --- a/contributor-book/book.toml +++ b/contributor-book/book.toml @@ -6,6 +6,7 @@ authors = [ "Dilshod Tadjibaev", "Guillaume Lagrange", "Joshua Ferguson", + "The Burn Community", ] language = "en" multilingual = false diff --git a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md index 89a1bdfa0b..a779e73488 100644 --- a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md +++ b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md @@ -1,6 +1,6 @@ # Issues encountered while adding ops -Most issues were fairly straight forward, and I was able to play whack-a-bug until I hit ones were I needed outside help (on the discord channels), so I'll share some of the ones that had me stumped enough to ask for help in case you too encounter them. +Below are some of the issues that were encountered while adding ops to the project. If you encounter an issue while adding an op that isn't listed here, and it's not obvious how to fix it, please add it to this list. ## Off by .000001 errors @@ -25,4 +25,4 @@ error[E0599]: no method named `powi` found for struct `Tensor` in the current sc For more information about an error, try `rustc --explain E0308`. error: could not compile `onnx-tests` (test "onnx_tests") due to 3 previous errors ``` -So if you are getting this, you probably didn't impl your operator for the actual Tensor struct. I wasn't aware of the existence or role of `burn-tensor/src/tensor/api/numeric.rs`, and had first defined just the Ops for the Int and Float tensors, that coupled with `powf` existing prior to the PR though for scalar values(which had been renamed, just not in the right place), led to this confusing issue where it looked like the function was found, but the type was wrong. If that's the case, make sure that it's implemented for the appropriate type, in this case `Float` under [burn-tensor/src/tensor/api/numeric.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/api/numeric.rs#L2186), and calling the `TensorOp.foo_op` defined under [burn-tensor/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/ops/tensor.rs#L873) +So if you are getting this, you probably didn't impl your operator for the actual Tensor struct. This issue was encountered when adding the Pow operator. The operation was added to the `FloatTensorOps` and `IntTensorOp` traits, but not for the numeric trait (under `burn-tensor/src/tensor/api/numeric.rs`). This, coupled with `powf` existing prior to the PR though only for scalar values (which had been renamed, just not in the right place), led to this confusing issue where it looked like the function was found, but the type was wrong. If that's the case, make sure that it's implemented for the appropriate type, in this case `Float` under [burn-tensor/src/tensor/api/numeric.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/api/numeric.rs#L2186), and calling the `TensorOp.foo_op` defined under [burn-tensor/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/ops/tensor.rs#L873) diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index a3418f6451..4be7180f54 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -15,9 +15,9 @@ There are a few commands you want to run prior to any commit for a non-draft PR: To bump for the next version, edit the semantic version number in `burn/Cargo.toml`, and then run `cargo update` to update the lock file. -## Contributing to the Burn (Contributor) Book +## Contributing to either the Burn Book or Contributor Book -Both the Burn book and the burn contributor book are built with mdbook. To install mdbook, run the following command[^1]: +Both the Burn book and the Contributor book are built with mdbook. To install mdbook, run the following command[^1]: ```bash cargo install mdbook From c4ee85b09ec1aaecf46ac5639b019d4220678d4c Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:31:07 -0600 Subject: [PATCH 19/32] fixed typography --- .../src/getting-started/setting-up-the-environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index 4be7180f54..ecba895a47 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -17,7 +17,7 @@ To bump for the next version, edit the semantic version number in `burn/Cargo.to ## Contributing to either the Burn Book or Contributor Book -Both the Burn book and the Contributor book are built with mdbook. To install mdbook, run the following command[^1]: +Both the Burn Book and the Contributor Book are built with mdbook. To install mdbook, run the following command[^1]: ```bash cargo install mdbook From f63a79dcef5f61cd808036698ce1b82742c72168 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 10:46:58 -0600 Subject: [PATCH 20/32] Apply suggestions from code review Co-authored-by: Sylvain Benner --- .../src/getting-started/setting-up-the-environment.md | 2 +- contributor-book/src/getting-started/testing.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index ecba895a47..7d495cf303 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -26,4 +26,4 @@ cargo install mdbook Also instead of running `./run_checks.sh all`, you can run `./run_checks.sh typo` to only check for misspellings. This will install [typo](https://crates.io/crates/typos-cli), and if any are encountered you should be able to run `typo -w /path/to/book` to fix them. [^1]: You might also want to install [cargo-update](https://github.com/nabijaczleweli/cargo-update) to easily keep your tools up to date, though it is in no way required. -[^2]: If your system is running into issues with memory and you are on linux you may want to switch to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console(and log in), and either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. \ No newline at end of file +[^2]: If your system is running into issues with memory and you are on linux, you may want to switch to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console (and log in), and either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. \ No newline at end of file diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index 540e531a10..32f8e1f352 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -2,13 +2,13 @@ ## Test for TensorOps -the following examples use matrix multiplication operation +The following examples use matrix multiplication operation. Test for Tensor operations (as in given this input, expect it match or approximate this output) are defined only in [`burn-tensor/src/test/ops`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/ops/matmul.rs#L1) and not in the backends, with the exception of `burn-autodiff`. These test are added to the `testgen_all` macro rule in [`burn-tensor/src/test/mod.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/mod.rs#L59). This is then propagated to the existing backends without any additional work. ### Test for Autodiff -the following examples use the power operation +The following examples use the power operation. Tests for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replace `foo` for whatever makes sense for your op), and for tensor operations both the left and right sides need to be verified. The easiest way to do this, is to: From 07db8ad01445cd6b03057e72eee126cc30b0095c Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Mon, 29 Jan 2024 15:57:19 -0600 Subject: [PATCH 21/32] fixed typos, and incorrect rendering of footnotes --- contributor-book/src/SUMMARY.md | 4 ++-- .../src/guides/adding-a-new-operation-to-burn.md | 6 ++++-- .../src/guides/frequently-encountered-issues/ReadMe.md | 1 + .../issues-while-adding-ops.md | 1 + 4 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 contributor-book/src/guides/frequently-encountered-issues/ReadMe.md create mode 100644 contributor-book/src/guides/frequently-encountered-issues/issues-while-adding-ops.md diff --git a/contributor-book/src/SUMMARY.md b/contributor-book/src/SUMMARY.md index 38c205e2e3..b96e20b627 100644 --- a/contributor-book/src/SUMMARY.md +++ b/contributor-book/src/SUMMARY.md @@ -12,5 +12,5 @@ - [Guides for Contributors](./guides/ReadMe.md) - [Onnx To Burn Conversion Tool: A Development Guide](./guides/onnx-to-burn-conversion-tool.md) - [Adding a New Operation to Burn](./guides/adding-a-new-operation-to-burn.md) - - [Frequently Encountered Issues](./guides/frequently-encountered-issues/ReadMe.md) - - [Issues Related To Adding Operators](./guides/frequently-encountered-issues/issues-while-adding-ops.md) \ No newline at end of file +- [Frequently Encountered Issues](./guides/frequently-encountered-issues/ReadMe.md) + - [Issues Related To Adding Operators](./guides/frequently-encountered-issues/issues-while-adding-ops.md) \ No newline at end of file diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index 8b2c2bb8ec..7ceb3f5650 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -20,7 +20,7 @@ The `Int` Tensor function use the ones defined for Float with 2 extra cast (LHS ### Adding Test -Additional Test should be added to `burn-tensor` under [`burn-tensor/src/tests/ops/{op_name}.rs`](https://github.com/tracel-ai/burn/burn-tensor/src/tests/ops/powf.rs), inserting the module name into ``burn-tensor/src/tests/ops/mod.rs`. Then add it to`testgen_all` macro under `burn-tensor/src/tests/mod.rs`. This test is automatically added to the backends so it isn't necessary to add them there, save for those that require specific testing such as`burn-autodiff` +Additional Test should be added to `burn-tensor` under [`burn-tensor/src/tests/ops/{op_name}.rs`](https://github.com/tracel-ai/burn/burn-tensor/src/tests/ops/powf.rs), inserting the module name into `burn-tensor/src/tests/ops/mod.rs`. Then add it to the `testgen_all` macro under `burn-tensor/src/tests/mod.rs`. This test is automatically added to the backends so it isn't necessary to add them there, save for those that require specific testing such as`burn-autodiff` ## Adding the Op to the burn-autodiff @@ -71,7 +71,7 @@ Adding an operator to these backends is fairly straightforward, though due to wh here's how powf was added to burn fusion: -1. added powf to the float ops under [burn-fusion/src/ops/float.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/ops/float.rs#L1758) +1. added powf to the float ops under [`burn-fusion/src/ops/float.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/ops/float.rs#L1758) 2. added powf to the `FloatOperationDescription` enum under [burn-fusion/src/stream/operation.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/stream/operation.rs#L385) 3. added powf to the implementations of `FloatOperationDescription` enum under [burn-fusion/src/stream/context.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/stream/context.rs#L726) @@ -101,5 +101,7 @@ Let's review the changes made for pow starting from `src/burn` and moving to `sr And you're done! Congrats, you just fully added a new op to burn, and we are all one step closer to the answer to [are we learning yet?](https://www.arewelearningyet.com/) being "Yes, and it's freaking fast!". Buy yourself a coffee [^1]: for more on supertraits see [the advanced trait section of the rust book](https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#using-supertraits-to-require-one-traits-functionality-within-another-trait) + [^2]: wiki link for [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) + [^3]: for more information on unit structs see [the defining and instantiating structs section of the rust book](https://doc.rust-lang.org/book/ch05-01-defining-structs.html#unit-like-structs-without-any-fields) diff --git a/contributor-book/src/guides/frequently-encountered-issues/ReadMe.md b/contributor-book/src/guides/frequently-encountered-issues/ReadMe.md new file mode 100644 index 0000000000..ce63065686 --- /dev/null +++ b/contributor-book/src/guides/frequently-encountered-issues/ReadMe.md @@ -0,0 +1 @@ +# Frequently Encountered Issues diff --git a/contributor-book/src/guides/frequently-encountered-issues/issues-while-adding-ops.md b/contributor-book/src/guides/frequently-encountered-issues/issues-while-adding-ops.md new file mode 100644 index 0000000000..e4d50fe577 --- /dev/null +++ b/contributor-book/src/guides/frequently-encountered-issues/issues-while-adding-ops.md @@ -0,0 +1 @@ +# Issues Related To Adding Operators From 1f8900713277c8b6c1573641a4f231e7a90c3de7 Mon Sep 17 00:00:00 2001 From: Dilshod Tadjibaev <939125+antimora@users.noreply.github.com> Date: Wed, 31 Jan 2024 13:06:21 -0600 Subject: [PATCH 22/32] Format the text using prettier --- contributor-book/src/SUMMARY.md | 2 +- .../issues-while-adding-ops.md | 26 ++- .../configuring-your-editor.md | 15 +- .../setting-up-the-environment.md | 33 ++- .../src/getting-started/testing.md | 21 +- .../guides/adding-a-new-operation-to-burn.md | 219 +++++++++++++----- contributor-book/src/how-to-read-this-book.md | 18 +- contributor-book/src/overview.md | 24 +- .../src/project-architecture/Tensor.md | 95 +++++--- .../src/project-architecture/backend.md | 36 +-- .../src/project-architecture/module.md | 105 +++++---- .../src/project-architecture/serialization.md | 109 +++++---- 12 files changed, 469 insertions(+), 234 deletions(-) diff --git a/contributor-book/src/SUMMARY.md b/contributor-book/src/SUMMARY.md index b96e20b627..ef5d4045c5 100644 --- a/contributor-book/src/SUMMARY.md +++ b/contributor-book/src/SUMMARY.md @@ -13,4 +13,4 @@ - [Onnx To Burn Conversion Tool: A Development Guide](./guides/onnx-to-burn-conversion-tool.md) - [Adding a New Operation to Burn](./guides/adding-a-new-operation-to-burn.md) - [Frequently Encountered Issues](./guides/frequently-encountered-issues/ReadMe.md) - - [Issues Related To Adding Operators](./guides/frequently-encountered-issues/issues-while-adding-ops.md) \ No newline at end of file + - [Issues Related To Adding Operators](./guides/frequently-encountered-issues/issues-while-adding-ops.md) diff --git a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md index a779e73488..a5181e8e88 100644 --- a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md +++ b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md @@ -1,6 +1,8 @@ # Issues encountered while adding ops -Below are some of the issues that were encountered while adding ops to the project. If you encounter an issue while adding an op that isn't listed here, and it's not obvious how to fix it, please add it to this list. +Below are some of the issues that were encountered while adding ops to the project. If you encounter +an issue while adding an op that isn't listed here, and it's not obvious how to fix it, please add +it to this list. ## Off by .000001 errors @@ -10,19 +12,29 @@ Below are some of the issues that were encountered while adding ops to the proje tests::maxmin::tests::test_mean_dim_2d stdout ---- thread 'tests::maxmin::tests::test_mean_dim_2d' panicked at burn-wgpu/src/lib.rs:49:5: assertion `left == right` failed left: Data { value: [1.0, 4.0], shape: Shape { dims: [2, 1] } } right: Data { value: [0.99999994, 3.9999998], shape: Shape { dims: [2, 1] } } ``` -If you encounter this, swap out the `assert_eq!` in the failing test for `tensor1.assert_approx_eq` with `3` as the second argument. +If you encounter this, swap out the `assert_eq!` in the failing test for `tensor1.assert_approx_eq` +with `3` as the second argument. ## Mismatched types and missing functions ```sh -error[E0308]: mismatched types --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow.rs:48:45 | 48 | let pow1_out1 = input1.clone().powf(input1); | ---- ^^^^^^ expected `f32`, found `Tensor` | | | arguments to this method are incorrect | = note: expected type `f32` found struct `Tensor` +error[E0308]: mismatched types --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow.rs:48:45 | 48 | let pow1_out1 = input1.clone().powf(input1); | ---- ^^^^^^ expected `f32`, found `Tensor` | | | arguments to this method are incorrect | = note: expected type `f32` found struct `Tensor` -note: method defined here --> {burn_dir}/burn-tensor/src/tensor/api/float.rs:65:12 | 65 | pub fn powf(self, value: f32) -> Self { | ^^^^ +note: method defined here --> {burn_dir}/burn-tensor/src/tensor/api/float.rs:65:12 | 65 | pub fn powf(self, value: f32) -> Self { | ^^^^ -error[E0599]: no method named `powf_scalar` found for struct `Tensor` in the current scope --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow.rs:50:35 | 50 | let pow2_out1 = pow1_out1.powf_scalar(cast1_out1); | ^^^^^^^^^^^ method not found in `Tensor` +error[E0599]: no method named `powf_scalar` found for struct `Tensor` in the current scope --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow.rs:50:35 | 50 | let pow2_out1 = pow1_out1.powf_scalar(cast1_out1); | ^^^^^^^^^^^ method not found in `Tensor` -error[E0599]: no method named `powi` found for struct `Tensor` in the current scope --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow_int.rs:49:40 | 49 | let pow1_out1 = input1.clone().powi(input1); | ^^^^ method not found in `Tensor` Some errors have detailed explanations: E0308, E0599. +error[E0599]: no method named `powi` found for struct `Tensor` in the current scope --> {burn_dir}/target/debug/build/onnx-tests-fed12aaf3671687f/out/model/pow_int.rs:49:40 | 49 | let pow1_out1 = input1.clone().powi(input1); | ^^^^ method not found in `Tensor` Some errors have detailed explanations: E0308, E0599. For more information about an error, try `rustc --explain E0308`. error: could not compile `onnx-tests` (test "onnx_tests") due to 3 previous errors ``` -So if you are getting this, you probably didn't impl your operator for the actual Tensor struct. This issue was encountered when adding the Pow operator. The operation was added to the `FloatTensorOps` and `IntTensorOp` traits, but not for the numeric trait (under `burn-tensor/src/tensor/api/numeric.rs`). This, coupled with `powf` existing prior to the PR though only for scalar values (which had been renamed, just not in the right place), led to this confusing issue where it looked like the function was found, but the type was wrong. If that's the case, make sure that it's implemented for the appropriate type, in this case `Float` under [burn-tensor/src/tensor/api/numeric.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/api/numeric.rs#L2186), and calling the `TensorOp.foo_op` defined under [burn-tensor/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/ops/tensor.rs#L873) +So if you are getting this, you probably didn't impl your operator for the actual Tensor struct. +This issue was encountered when adding the Pow operator. The operation was added to the +`FloatTensorOps` and `IntTensorOp` traits, but not for the numeric trait (under +`burn-tensor/src/tensor/api/numeric.rs`). This, coupled with `powf` existing prior to the PR though +only for scalar values (which had been renamed, just not in the right place), led to this confusing +issue where it looked like the function was found, but the type was wrong. If that's the case, make +sure that it's implemented for the appropriate type, in this case `Float` under +[burn-tensor/src/tensor/api/numeric.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/api/numeric.rs#L2186), +and calling the `TensorOp.foo_op` defined under +[burn-tensor/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tensor/src/tensor/ops/tensor.rs#L873) diff --git a/contributor-book/src/getting-started/configuring-your-editor.md b/contributor-book/src/getting-started/configuring-your-editor.md index fe823a53df..5d3657ce3a 100644 --- a/contributor-book/src/getting-started/configuring-your-editor.md +++ b/contributor-book/src/getting-started/configuring-your-editor.md @@ -1,6 +1,7 @@ # Configuring your editor -These are not required, and most of this isn't specific to Burn, but it's definitely helpful if you haven't already done it. +These are not required, and most of this isn't specific to Burn, but it's definitely helpful if you +haven't already done it. ## VSCode @@ -11,14 +12,18 @@ These are not required, and most of this isn't specific to Burn, but it's defini - [serayuzgur.crates](https://marketplace.visualstudio.com/items?itemName=serayuzgur.crates) - [vadimcn.vscode-lldb](https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb) -2. Open `Command Palette` with Ctrl+Shift+P or F1 and type `LLDB: Generate Launch Configurations from Cargo.toml` then select it, this will generate a file that should be saved as `.vscode/launch.json`. +2. Open `Command Palette` with Ctrl+Shift+P or F1 and type + `LLDB: Generate Launch Configurations from Cargo.toml` then select it, this will generate a file + that should be saved as `.vscode/launch.json`. -3. Now you can enable breakpoint on code through IDE and then start debugging the library/binary you want, such as the following example: +3. Now you can enable breakpoint on code through IDE and then start debugging the library/binary you + want, such as the following example:
-4. If you're creating a new library or binary, keep in mind to repeat the step 2 to always keep a fresh list of targets. +4. If you're creating a new library or binary, keep in mind to repeat the step 2 to always keep a + fresh list of targets. -## Have another editor? Open a PR! \ No newline at end of file +## Have another editor? Open a PR! diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index 7d495cf303..5975a7c857 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -1,29 +1,44 @@ # setting up the environment -There are a couple of tools that need to be installed, and commands to be familiar with, depending on what part of the project you plan on contributing to. This section should be up to date with current project practices (as of 2024-01-26) +There are a couple of tools that need to be installed, and commands to be familiar with, depending +on what part of the project you plan on contributing to. This section should be up to date with +current project practices (as of 2024-01-26) ## General There are a few commands you want to run prior to any commit for a non-draft PR: -1. `cargo clippy --fix --allow-dirty`, this will run clippy and fix any issues it can, the allow dirty flag is required whenever you have uncommitted changes +1. `cargo clippy --fix --allow-dirty`, this will run clippy and fix any issues it can, the allow + dirty flag is required whenever you have uncommitted changes 2. `cargo fmt --all`, this will run rustfmt on all files in the project -3. `./run_checks.sh all`, this is a script located in the project root that builds and tests the project. It is required that this passes prior to merging a PR. Fair warning, running these tests can take a while[^2]. +3. `./run_checks.sh all`, this is a script located in the project root that builds and tests the + project. It is required that this passes prior to merging a PR. Fair warning, running these tests + can take a while[^2]. ## Updating the burn semver version -To bump for the next version, edit the semantic version number in `burn/Cargo.toml`, and then run `cargo update` to update the lock file. - +To bump for the next version, edit the semantic version number in `burn/Cargo.toml`, and then run +`cargo update` to update the lock file. ## Contributing to either the Burn Book or Contributor Book -Both the Burn Book and the Contributor Book are built with mdbook. To install mdbook, run the following command[^1]: +Both the Burn Book and the Contributor Book are built with mdbook. To install mdbook, run the +following command[^1]: ```bash cargo install mdbook ``` -Also instead of running `./run_checks.sh all`, you can run `./run_checks.sh typo` to only check for misspellings. This will install [typo](https://crates.io/crates/typos-cli), and if any are encountered you should be able to run `typo -w /path/to/book` to fix them. +Also instead of running `./run_checks.sh all`, you can run `./run_checks.sh typo` to only check for +misspellings. This will install [typo](https://crates.io/crates/typos-cli), and if any are +encountered you should be able to run `typo -w /path/to/book` to fix them. + +[^1]: + You might also want to install [cargo-update](https://github.com/nabijaczleweli/cargo-update) to + easily keep your tools up to date, though it is in no way required. -[^1]: You might also want to install [cargo-update](https://github.com/nabijaczleweli/cargo-update) to easily keep your tools up to date, though it is in no way required. -[^2]: If your system is running into issues with memory and you are on linux, you may want to switch to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console (and log in), and either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. \ No newline at end of file +[^2]: + If your system is running into issues with memory and you are on linux, you may want to switch + to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run + the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console (and log in), and + either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index 32f8e1f352..6d8ba5d702 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -4,16 +4,29 @@ The following examples use matrix multiplication operation. -Test for Tensor operations (as in given this input, expect it match or approximate this output) are defined only in [`burn-tensor/src/test/ops`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/ops/matmul.rs#L1) and not in the backends, with the exception of `burn-autodiff`. These test are added to the `testgen_all` macro rule in [`burn-tensor/src/test/mod.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/mod.rs#L59). This is then propagated to the existing backends without any additional work. +Test for Tensor operations (as in given this input, expect it match or approximate this output) are +defined only in +[`burn-tensor/src/test/ops`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/ops/matmul.rs#L1) +and not in the backends, with the exception of `burn-autodiff`. These test are added to the +`testgen_all` macro rule in +[`burn-tensor/src/test/mod.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/mod.rs#L59). +This is then propagated to the existing backends without any additional work. ### Test for Autodiff The following examples use the power operation. -Tests for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replace `foo` for whatever makes sense for your op), and for tensor operations both the left and right sides need to be verified. The easiest way to do this, is to: +Tests for autodiff go under +[burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) +(replace `foo` for whatever makes sense for your op), and for tensor operations both the left and +right sides need to be verified. The easiest way to do this, is to: 1. use small tensors with simple values -2. pop open a terminal, launch `ipython` and import `numpy` then do the calculations by hand. You can also use [google colab](https://colab.google/) if you prefer so that you don't have to install the packages on your system. +2. pop open a terminal, launch `ipython` and import `numpy` then do the calculations by hand. You + can also use [google colab](https://colab.google/) if you prefer so that you don't have to + install the packages on your system. 3. compare the actual output to the expected output for lhs, rhs and regular operation -generally, it seems preferable to use `actual_output_tensor.into_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. +generally, it seems preferable to use +`actual_output_tensor.into_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due +to occasional hiccups with floating point calculations. diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index 7ceb3f5650..9b7d4c2ea8 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -1,107 +1,218 @@ # Adding a new operation to burn -Let's discuss how one might go about adding new operators to Burn, using the example of the recently added [pow operator](https://github.com/tracel-ai/burn/pull/1133/files) (as of 01/24/2024). In that PR, the following things took place (albeit not in this order) +Let's discuss how one might go about adding new operators to Burn, using the example of the recently +added [pow operator](https://github.com/tracel-ai/burn/pull/1133/files) (as of 01/24/2024). In that +PR, the following things took place (albeit not in this order) ## Adding the Op to burn-tensor -`burn-tensor` is the crate that defines all tensor operations that need to be implemented by the various backends. The core of this lies in `burn-tensor/src/tensor/api/numeric.rs`, which is home to the numeric trait and its implementation for the different tensor types. The numeric trait is the home of all tensor operations that are numeric in nature and that are shared by `Int` and `Float` Tensor types. More information on the relationship between Tensor modules can be found under the section for [Tensor Architecture](../project-architecture/Tensor.md#tensorops). +`burn-tensor` is the crate that defines all tensor operations that need to be implemented by the +various backends. The core of this lies in `burn-tensor/src/tensor/api/numeric.rs`, which is home to +the numeric trait and its implementation for the different tensor types. The numeric trait is the +home of all tensor operations that are numeric in nature and that are shared by `Int` and `Float` +Tensor types. More information on the relationship between Tensor modules can be found under the +section for [Tensor Architecture](../project-architecture/Tensor.md#tensorops). here is where pow was added to `burn-tensor/src/tensor/api/numeric.rs`: -1. for the [`Tensor` struct](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L553) -2. for the [numeric trait](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1618) -3. for the impl of numeric for [float](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L2186) and [int](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1903) - -Tensor is a struct that has a single member: `primitive`(defined [here](https://github.com/tracel-ai/burn/blob/main/burn-tensor/src/tensor/api/base.rs)), That is defined by it's [`Kind`](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/kind.rs#L15): one of `Bool`, `Float`, or `Int` (those linked in 3). These call the ops for that data type defined in the [`Backend`](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/backend/base.rs#L52) supertrait[^1]. This is the trait that is then implemented by the different `burn-` back-ends (such as `burn-ndarray` and `burn-wgpu`) which implement the functions if no default is provided. - -In this case, we only don't need to worry about `Bool` Tensors. Ops for `Float` is implemented under [burn-tensor/src/tensor/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/ops/tensor.rs#L873), and for `Int` under [`burn-tensor/src/tensor/ops/int_tensor.rs`](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-tensor/src/tensor/ops/int_tensor.rs#L486). The current convention is for int and bool ops, to be prefixed with `int` or `bool`, respectively. As we need to use floating point powers to ensure precision, the `powi` functions are given a default implementation that converts the right hand side of the operator to a float. - -The `Int` Tensor function use the ones defined for Float with 2 extra cast (LHS to a `Float` tensor, Output to an `Int`). Given that the rest of the code will only look at the float implementations. +1. for the + [`Tensor` struct](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L553) +2. for the + [numeric trait](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1618) +3. for the impl of numeric for + [float](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L2186) + and + [int](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1903) + +Tensor is a struct that has a single member: `primitive`(defined +[here](https://github.com/tracel-ai/burn/blob/main/burn-tensor/src/tensor/api/base.rs)), That is +defined by it's +[`Kind`](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/kind.rs#L15): +one of `Bool`, `Float`, or `Int` (those linked in 3). These call the ops for that data type defined +in the +[`Backend`](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/backend/base.rs#L52) +supertrait[^1]. This is the trait that is then implemented by the different `burn-` back-ends (such +as `burn-ndarray` and `burn-wgpu`) which implement the functions if no default is provided. + +In this case, we only don't need to worry about `Bool` Tensors. Ops for `Float` is implemented under +[burn-tensor/src/tensor/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/ops/tensor.rs#L873), +and for `Int` under +[`burn-tensor/src/tensor/ops/int_tensor.rs`](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-tensor/src/tensor/ops/int_tensor.rs#L486). +The current convention is for int and bool ops, to be prefixed with `int` or `bool`, respectively. +As we need to use floating point powers to ensure precision, the `powi` functions are given a +default implementation that converts the right hand side of the operator to a float. + +The `Int` Tensor function use the ones defined for Float with 2 extra cast (LHS to a `Float` tensor, +Output to an `Int`). Given that the rest of the code will only look at the float implementations. ### Adding Test -Additional Test should be added to `burn-tensor` under [`burn-tensor/src/tests/ops/{op_name}.rs`](https://github.com/tracel-ai/burn/burn-tensor/src/tests/ops/powf.rs), inserting the module name into `burn-tensor/src/tests/ops/mod.rs`. Then add it to the `testgen_all` macro under `burn-tensor/src/tests/mod.rs`. This test is automatically added to the backends so it isn't necessary to add them there, save for those that require specific testing such as`burn-autodiff` +Additional Test should be added to `burn-tensor` under +[`burn-tensor/src/tests/ops/{op_name}.rs`](https://github.com/tracel-ai/burn/burn-tensor/src/tests/ops/powf.rs), +inserting the module name into `burn-tensor/src/tests/ops/mod.rs`. Then add it to the `testgen_all` +macro under `burn-tensor/src/tests/mod.rs`. This test is automatically added to the backends so it +isn't necessary to add them there, save for those that require specific testing such +as`burn-autodiff` ## Adding the Op to the burn-autodiff -Since this is probably the hardest and the least straightforward, we'll cover this backend separately. Burn-autodiff enables other backends to use autodifferentiation[^2]. Ops for float types are implemented in [burn-autodiff/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-autodiff/src/ops/tensor.rs#L1523) and need to: +Since this is probably the hardest and the least straightforward, we'll cover this backend +separately. Burn-autodiff enables other backends to use autodifferentiation[^2]. Ops for float types +are implemented in +[burn-autodiff/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-autodiff/src/ops/tensor.rs#L1523) +and need to: 1. define a unit struct [^3] that implements a backward (pass) function -2. Within the backward function, as this is an elementwise binary operation it implements the binary function (from backward.rs under the same directory), the last 2 arguments are two closures that define the left and right partial derivatives. -3. Then defines what happens when a specific operation is tracked or untracked, where untracked just calls the function in the normal way, and tracked executes the backward function defined above +2. Within the backward function, as this is an elementwise binary operation it implements the binary + function (from backward.rs under the same directory), the last 2 arguments are two closures that + define the left and right partial derivatives. +3. Then defines what happens when a specific operation is tracked or untracked, where untracked just + calls the function in the normal way, and tracked executes the backward function defined above -steps 1 and 3 are boilerplate, so much so that you can probably just copy the contents of another op of the same type (binary, unary) and change the name of the struct, and ensure that either both sides have the data they need (if they need to have a copy of the opposite sided tensor, clone it's contents). +steps 1 and 3 are boilerplate, so much so that you can probably just copy the contents of another op +of the same type (binary, unary) and change the name of the struct, and ensure that either both +sides have the data they need (if they need to have a copy of the opposite sided tensor, clone it's +contents). -now for step 2. Since a significant number of the people reading this probably haven't touched calculus either ever, or since however long ago you took the appropriate course, I'll assume that you, the observer, have some prior knowledge of calculus but would benefit from a review of the concepts. If this is not the case, I apologize, you can probably skim this section. +now for step 2. Since a significant number of the people reading this probably haven't touched +calculus either ever, or since however long ago you took the appropriate course, I'll assume that +you, the observer, have some prior knowledge of calculus but would benefit from a review of the +concepts. If this is not the case, I apologize, you can probably skim this section. -In the case of pow, since this is a binary operation, the left and right functions are the partial derivatives with respect to the left and right sided tensors. +In the case of pow, since this is a binary operation, the left and right functions are the partial +derivatives with respect to the left and right sided tensors. -Let's define the operator as a function \\(f(x,y)=x^{y}\\) , where \\(x\\) is the left hand tensor and \\(y\\) is the right handed tensor. The two closures are defining the partial derivatives of \\(f\\) with respect to \\(x\\),\\(y\\). The eli5 is treat the other variable as a constant +Let's define the operator as a function \\(f(x,y)=x^{y}\\) , where \\(x\\) is the left hand tensor +and \\(y\\) is the right handed tensor. The two closures are defining the partial derivatives of +\\(f\\) with respect to \\(x\\),\\(y\\). The eli5 is treat the other variable as a constant -$$\frac{\delta }{\delta x} (x^{y})= y \cdot x^{y-1}$$ -is the left handed closure, and +$$\frac{\delta }{\delta x} (x^{y})= y \cdot x^{y-1}$$ is the left handed closure, and $$\frac{\delta }{\delta y} (x^{y}) = x^{y} \cdot ln(x)$$ -is the right. If you aren't sure how to calculate these by hand, I recommend using [symbolab](https://www.symbolab.com/solver/partial-derivative-calculator/%5Cfrac%7B%5Cpartial%7D%7B%5Cpartial%20x%7D%5Cleft(x%5E%7By%7D%5Cright)?or=input), plug in your operator in terms of \\(x\\) and \\(y\\), and just swap out the variable \\(x\\)|\\(y\\) in the partial derivative to get the other side. +is the right. If you aren't sure how to calculate these by hand, I recommend using +[symbolab](), +plug in your operator in terms of \\(x\\) and \\(y\\), and just swap out the variable +\\(x\\)|\\(y\\) in the partial derivative to get the other side. ### Testing autodiff -Test for autodiff go under [burn-autodiff/src/tests/foo.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) (replacing foo for whatever makes sense for your op), and for tensor operations both the left and right side need to be verified. The easiest way to do this, is to +Test for autodiff go under +[burn-autodiff/src/tests/foo.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) +(replacing foo for whatever makes sense for your op), and for tensor operations both the left and +right side need to be verified. The easiest way to do this, is to 1. use small tensors with simple values -2. pop open a terminal and launch `ipython` import `numpy` (or just use [google colab](https://colab.google/) if you don't have the packages installed and don't want to install them), and do the calculations by hand. +2. pop open a terminal and launch `ipython` import `numpy` (or just use + [google colab](https://colab.google/) if you don't have the packages installed and don't want to + install them), and do the calculations by hand. 3. comparing the actual to expected output for lhs, rhs and regular operation -generally, it seems preferable to use `actual_output_tensor.to_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. +generally, it seems preferable to use +`actual_output_tensor.to_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due +to occasional hiccups with floating point calculations. ## Adding the Op to other backends -most of these are fairly straightforward implementations. for reference here's pow's float implementation for torch, ndarray and candle backends: +most of these are fairly straightforward implementations. for reference here's pow's float +implementation for torch, ndarray and candle backends: -1. Torch implementation in [burn-tch/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/main/burn-tch/src/ops/tensor.rs#L461) and the Op used in [burn-tch/src/ops/base.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tch/src/ops/base.rs#L443) -2. NdArray in [burn-ndarray/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/main/burn-ndarray/src/ops/tensor.rs#L443) -3. Candle in [burn-candle/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-candle/src/ops/tensor.rs#L481) +1. Torch implementation in + [burn-tch/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/main/burn-tch/src/ops/tensor.rs#L461) + and the Op used in + [burn-tch/src/ops/base.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-tch/src/ops/base.rs#L443) +2. NdArray in + [burn-ndarray/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/main/burn-ndarray/src/ops/tensor.rs#L443) +3. Candle in + [burn-candle/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-candle/src/ops/tensor.rs#L481) -This is where any calculation happens currently. Playing a guessing game with method names and seeing what completions are suggested will take you far. If you are having trouble figuring out how to do it from the docs for that backend, [try searching github for relevant function calls](https://docs.github.com/en/search-github/github-code-search/understanding-github-code-search-syntax). +This is where any calculation happens currently. Playing a guessing game with method names and +seeing what completions are suggested will take you far. If you are having trouble figuring out how +to do it from the docs for that backend, +[try searching github for relevant function calls](https://docs.github.com/en/search-github/github-code-search/understanding-github-code-search-syntax). ## Adding the Op to fusion and wgpu backends -Adding an operator to these backends is fairly straightforward, though due to what these backends are for, involves a bit more indirection. Fusion, like autodiff, is not a target backend as much as a backend that enables certain functionality for other backends, in this case kernel fusion (which is currently only supported for `burn-wgpu`), so adding the operator won't involve doing any calculation, you'll just be describing how the generated code should look. Most of this can be copy/pasted/adjusted from other functions. +Adding an operator to these backends is fairly straightforward, though due to what these backends +are for, involves a bit more indirection. Fusion, like autodiff, is not a target backend as much as +a backend that enables certain functionality for other backends, in this case kernel fusion (which +is currently only supported for `burn-wgpu`), so adding the operator won't involve doing any +calculation, you'll just be describing how the generated code should look. Most of this can be +copy/pasted/adjusted from other functions. here's how powf was added to burn fusion: -1. added powf to the float ops under [`burn-fusion/src/ops/float.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/ops/float.rs#L1758) -2. added powf to the `FloatOperationDescription` enum under [burn-fusion/src/stream/operation.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/stream/operation.rs#L385) -3. added powf to the implementations of `FloatOperationDescription` enum under [burn-fusion/src/stream/context.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/stream/context.rs#L726) +1. added powf to the float ops under + [`burn-fusion/src/ops/float.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/ops/float.rs#L1758) +2. added powf to the `FloatOperationDescription` enum under + [burn-fusion/src/stream/operation.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/stream/operation.rs#L385) +3. added powf to the implementations of `FloatOperationDescription` enum under + [burn-fusion/src/stream/context.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/stream/context.rs#L726) - Adding pow to wgpu was actually pretty easy due to the design. Element-wise tensor ops are just vectorized scalar ops, and given that raising a tensor to a scalar power prior to the tensor version, I just reused the code for scalar powf. +Adding pow to wgpu was actually pretty easy due to the design. Element-wise tensor ops are just +vectorized scalar ops, and given that raising a tensor to a scalar power prior to the tensor +version, I just reused the code for scalar powf. here is where code was added -1. to the implementation of [`TensorOps` under `burn-wgpu/src/ops/float_ops.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/float_ops.rs#L513) -2. the function being called was added to [burn-wgpu/src/ops/numeric.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/numeric.rs#L199) -3. the call to the fmt function use to generate wgsl code in [`burn-wgpu/src/codegen/kernel.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/codegen/kernel.rs#L208) -4. A custom function generator was added to [`burn-wgpu/src/codegen/function.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/codegen/function.rs#L99) - -Much of the logic for powf had already been defined, so not much needed to be added. The reason for the custom powf function is generated, rather than directly using the underlying function as is the case with other operations, is due to issues with case handling of the wgsl pow function, like 0 to the 0 power being 1, and any negative number to an even power being positive. We reused as much as the existing logic as possible, such as the operation output description generation in [`burn-wgpu/src/fusion/elemwise/builder.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/fusion/elemwise/optimization.rs) and then branched at the last point based off the var type of the rhs. I don't know if there are other places within `burn-wgpu` where additions would have been necessary otherwise. +1. to the implementation of + [`TensorOps` under `burn-wgpu/src/ops/float_ops.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/float_ops.rs#L513) +2. the function being called was added to + [burn-wgpu/src/ops/numeric.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/numeric.rs#L199) +3. the call to the fmt function use to generate wgsl code in + [`burn-wgpu/src/codegen/kernel.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/codegen/kernel.rs#L208) +4. A custom function generator was added to + [`burn-wgpu/src/codegen/function.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/codegen/function.rs#L99) + +Much of the logic for powf had already been defined, so not much needed to be added. The reason for +the custom powf function is generated, rather than directly using the underlying function as is the +case with other operations, is due to issues with case handling of the wgsl pow function, like 0 to +the 0 power being 1, and any negative number to an even power being positive. We reused as much as +the existing logic as possible, such as the operation output description generation in +[`burn-wgpu/src/fusion/elemwise/builder.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/fusion/elemwise/optimization.rs) +and then branched at the last point based off the var type of the rhs. I don't know if there are +other places within `burn-wgpu` where additions would have been necessary otherwise. ## Adding the Op to burn-import -I won't comment on generating the test onnx files or the test, as that is already covered [in the ONNX to burn guide](onnx-to-burn-conversion-tool.md#adding-new-operators), this is more about the specific changes you need to make when adding new operators after you have generated the tests. +I won't comment on generating the test onnx files or the test, as that is already covered +[in the ONNX to burn guide](onnx-to-burn-conversion-tool.md#adding-new-operators), this is more +about the specific changes you need to make when adding new operators after you have generated the +tests. -The crate is divided into two sections `src/burn` and `src/onnx`. The code under the former corresponds to the operation you've implemented earlier in this guide, and the latter to the operations defined in the onnx specification. So when you are loading a model, the operator is first parsed to an intermediate representation defined by `src/onnx`, and then mapped to a Burn operations defined under `src/burn/node`. +The crate is divided into two sections `src/burn` and `src/onnx`. The code under the former +corresponds to the operation you've implemented earlier in this guide, and the latter to the +operations defined in the onnx specification. So when you are loading a model, the operator is first +parsed to an intermediate representation defined by `src/onnx`, and then mapped to a Burn operations +defined under `src/burn/node`. Let's review the changes made for pow starting from `src/burn` and moving to `src/onnx`: -1. determine the type of operator and add your operator to the appropriate node (operation) type, in this case [BinaryNode under burn-import/src/burn/node/binary.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/burn/node/binary.rs#L160) along with its [`to_str` definition](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/burn/node/binary.rs#L9) -2. add an arm to the match statement inside the `into_burn` function in [burn-import/src/onnx/to_burn.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/to_burn.rs#L269) for the onnx `NodeType`(corresponds to an op in the Onnx spec), and make a [`foo_conversion` function](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/to_burn.rs#L667) that maps the onnx node to the binary type -3. specify how dimensions for the output should be derived in [burn-import/src/onnx/dim_inference.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/dim_inference.rs#L103) - -And you're done! Congrats, you just fully added a new op to burn, and we are all one step closer to the answer to [are we learning yet?](https://www.arewelearningyet.com/) being "Yes, and it's freaking fast!". Buy yourself a coffee - -[^1]: for more on supertraits see [the advanced trait section of the rust book](https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#using-supertraits-to-require-one-traits-functionality-within-another-trait) - -[^2]: wiki link for [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) - -[^3]: for more information on unit structs see [the defining and instantiating structs section of the rust book](https://doc.rust-lang.org/book/ch05-01-defining-structs.html#unit-like-structs-without-any-fields) +1. determine the type of operator and add your operator to the appropriate node (operation) type, in + this case + [BinaryNode under burn-import/src/burn/node/binary.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/burn/node/binary.rs#L160) + along with its + [`to_str` definition](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/burn/node/binary.rs#L9) +2. add an arm to the match statement inside the `into_burn` function in + [burn-import/src/onnx/to_burn.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/to_burn.rs#L269) + for the onnx `NodeType`(corresponds to an op in the Onnx spec), and make a + [`foo_conversion` function](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/to_burn.rs#L667) + that maps the onnx node to the binary type +3. specify how dimensions for the output should be derived in + [burn-import/src/onnx/dim_inference.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/dim_inference.rs#L103) + +And you're done! Congrats, you just fully added a new op to burn, and we are all one step closer to +the answer to [are we learning yet?](https://www.arewelearningyet.com/) being "Yes, and it's +freaking fast!". Buy yourself a coffee + +[^1]: + for more on supertraits see + [the advanced trait section of the rust book](https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#using-supertraits-to-require-one-traits-functionality-within-another-trait) + +[^2]: + wiki link for + [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) + +[^3]: + for more information on unit structs see + [the defining and instantiating structs section of the rust book](https://doc.rust-lang.org/book/ch05-01-defining-structs.html#unit-like-structs-without-any-fields) diff --git a/contributor-book/src/how-to-read-this-book.md b/contributor-book/src/how-to-read-this-book.md index 9a5c4a7c7f..2b0197ded8 100644 --- a/contributor-book/src/how-to-read-this-book.md +++ b/contributor-book/src/how-to-read-this-book.md @@ -4,11 +4,19 @@ Throughout this book, we try to keep the following structure ## Linking -When referring to structures or functions within codebase, we provide permalinks to the lines in specific commits, and indicate them by the relative path of their parent file from the project root. For example this is a reference to the `Tensor` struct in [`burn-tensor/src/tensor/api/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tensor/api/base.rs#L23) +When referring to structures or functions within codebase, we provide permalinks to the lines in +specific commits, and indicate them by the relative path of their parent file from the project root. +For example this is a reference to the `Tensor` struct in +[`burn-tensor/src/tensor/api/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tensor/api/base.rs#L23) -When some reference information is useful but is beyond the scope of contributing to burn, we provide that information in a footnote. To build on the previous example, the `Tensor` mentioned is what's referred to as a newtype struct[^1]. +When some reference information is useful but is beyond the scope of contributing to burn, we +provide that information in a footnote. To build on the previous example, the `Tensor` mentioned is +what's referred to as a newtype struct[^1]. -Direct hyperlinks are for tools and resources that are not part of the burn project, but are useful for contributing to it. For example, when working on implementing an op for autodiff, it is useful to use [symbolab](https://www.symbolab.com/) to calculate the left and right partial derivatives. - -[^1]: for more information on newtype please refer to [the Advanced Types chapter of the Rust Book](https://doc.rust-lang.org/book/ch19-04-advanced-types.html#using-the-newtype-pattern-for-type-safety-and-abstraction) +Direct hyperlinks are for tools and resources that are not part of the burn project, but are useful +for contributing to it. For example, when working on implementing an op for autodiff, it is useful +to use [symbolab](https://www.symbolab.com/) to calculate the left and right partial derivatives. +[^1]: + for more information on newtype please refer to + [the Advanced Types chapter of the Rust Book](https://doc.rust-lang.org/book/ch19-04-advanced-types.html#using-the-newtype-pattern-for-type-safety-and-abstraction) diff --git a/contributor-book/src/overview.md b/contributor-book/src/overview.md index f6e4f6ece2..8edff02a15 100644 --- a/contributor-book/src/overview.md +++ b/contributor-book/src/overview.md @@ -2,16 +2,24 @@ Welcome to The Burn Contributor's Book 👋 -This book will help you get acquainted with the internals of the Burn deep learning framework and provide some guidance on how to contribute to the project. +This book will help you get acquainted with the internals of the Burn deep learning framework and +provide some guidance on how to contribute to the project. - We have crafted some sections for you: +We have crafted some sections for you: -- [Getting Started](./getting-started): Much like the Burn Book for users, we'll start with the fundamentals, guiding you through tasks like setting up the development environment, how to run tests, and what you should check prior to each commit. - -- [Project Architecture](./project-architecture): This section will give you a more in-depth look at the architecture of burn +- [Getting Started](./getting-started): Much like the Burn Book for users, we'll start with the + fundamentals, guiding you through tasks like setting up the development environment, how to run + tests, and what you should check prior to each commit. +- [Project Architecture](./project-architecture): This section will give you a more in-depth look at + the architecture of burn -- [Guides](./guides): We'll provide some guides on how to do specific tasks, such as adding a new operation to Burn. +- [Guides](./guides): We'll provide some guides on how to do specific tasks, such as adding a new + operation to Burn. -- [Frequently Encountered Issues](./frequently-encountered-issues): If you are running into an issue that has you stumped, this is the section to check out prior to asking on the discord. It's a collection of errors encountered by contributors, what caused them, and how they were resolved. +- [Frequently Encountered Issues](./frequently-encountered-issues): If you are running into an issue + that has you stumped, this is the section to check out prior to asking on the discord. It's a + collection of errors encountered by contributors, what caused them, and how they were resolved. -As this book is geared more towards contributors rather than users of burn, we'll assume you have a good understanding of software development, but will make efforts to explain anything outside of that scope, or at least provide links to resources that explain it better than we can. +As this book is geared more towards contributors rather than users of burn, we'll assume you have a +good understanding of software development, but will make efforts to explain anything outside of +that scope, or at least provide links to resources that explain it better than we can. diff --git a/contributor-book/src/project-architecture/Tensor.md b/contributor-book/src/project-architecture/Tensor.md index 0a79248652..3913e241b8 100644 --- a/contributor-book/src/project-architecture/Tensor.md +++ b/contributor-book/src/project-architecture/Tensor.md @@ -1,43 +1,62 @@ # Tensor -A proper deep learning framework should have a fast tensor implementation with autodiff support, and Burn is no exception. -The tensor API abstracts away backend implementation details and focuses on usability without compromising performance. -To make it as easy as possible to use, there is only one tensor type, which is different from multiple tensor and deep learning crates in Rust. -Generic parameters are used instead to specialize the tensor type. - -- **B: Backend:** - The first argument is the backend on which the tensor implementation lies. -- **const D: usize:** - The second argument is the dimensionality of the tensor. -- **K: TensorKind:** - The third argument is the tensor kind, which can be either Float, Int or Bool. - By default, the tensor kind is set to Float, so for most tensors, the kind argument is not necessary. - -Having one struct for tensors reduces the complexity of the tensor API, which also means less duplicated documentation to write and maintain. - -Tensors are thread-safe, which means that you can send a tensor to another thread, and everything will work, including auto-differentiation. -Note that there are no in-place tensor operations since all tensor operations take owned tensors as parameters, which make it possible to mutate them. -Tensors can be shared simply by cloning them, but if there is only one reference to a tensor, the backend implementation is free to reuse the tensor's allocated data. -For more information about how it is done, you can have a look at this [blog post](https://burn.dev/blog/burn-rusty-approach-to-tensor-handling). +A proper deep learning framework should have a fast tensor implementation with autodiff support, and +Burn is no exception. The tensor API abstracts away backend implementation details and focuses on +usability without compromising performance. To make it as easy as possible to use, there is only one +tensor type, which is different from multiple tensor and deep learning crates in Rust. Generic +parameters are used instead to specialize the tensor type. + +- **B: Backend:** The first argument is the backend on which the tensor implementation lies. +- **const D: usize:** The second argument is the dimensionality of the tensor. +- **K: TensorKind:** The third argument is the tensor kind, which can be either Float, Int or Bool. + By default, the tensor kind is set to Float, so for most tensors, the kind argument is not + necessary. + +Having one struct for tensors reduces the complexity of the tensor API, which also means less +duplicated documentation to write and maintain. + +Tensors are thread-safe, which means that you can send a tensor to another thread, and everything +will work, including auto-differentiation. Note that there are no in-place tensor operations since +all tensor operations take owned tensors as parameters, which make it possible to mutate them. +Tensors can be shared simply by cloning them, but if there is only one reference to a tensor, the +backend implementation is free to reuse the tensor's allocated data. For more information about how +it is done, you can have a look at this +[blog post](https://burn.dev/blog/burn-rusty-approach-to-tensor-handling). ## TensorOps -Operations on Tensors are defined in traits (generally part of the Backend Supertrait) and implemented for the Tensor struct. The appropriate parent trait of an op depends on the type of operation: - -- `base` => All tensor kinds should implement these operations (Reshape, into_data, etc.). The implementation is in `burn-tensor/src/tensor/api/base.rs`. -- `numeric` => All tensors that are numeric by nature should implement these operations (Add, Sub, Div, etc.). The implementation is in `burn-tensor/src/tensor/api/numeric.rs`. -- `Float` => Tensor operations are only available for float tensors. The implementation is in `burn-tensor/src/tensor/api/float.rs`. -- `Int` => Tensor operations are only available for int tensors. The implementation is in `burn-tensor/src/tensor/api/int.rs`. -- `bool` => Tensor operations are only available for bool tensors. The implementation is in `burn-tensor/src/tensor/api/bool.rs`. - -`Numeric` is directly implemented for `Float` and `Int` tensors, and in general, The implementations for these methods are calling the corresponding `{Int|Float}TensorOp` method defined in the backend supertrait. - -Anything that is implemented by numeric should have an implementation in the `{Int|Float}TensorOp` traits, though it may be avoidable if the operation for one type requires casting to the other type. To provide an example, Powf should be implemented for `Int` tensors, but it should not be an Int Tensor Operation. The LHS should be converted to a float, and the output should be converted back to an int. So it's possible to avoid implementing `IntTensorOp` altogether. - -additionally there are some operations that should be defined as functions instead of tensor/tensor op methods. these are: - -`module` => These should be exported as functions instead of methods on tensors. The implementation is in `burn-tensor/src/tensor/module.rs` (Might be moved to `tensor/api/module.rs`). -`activation` => These should also be exported as functions instead of methods on tensors. The implementation is in `burn-tensor/src/tensor/activation/base.rs` (Might be moved to `tensor/api/activation.rs`). - -Note that some activations are just a combination of backend operations and are not declared in `burn-tensor/src/tensor/ops/activation.rs` - +Operations on Tensors are defined in traits (generally part of the Backend Supertrait) and +implemented for the Tensor struct. The appropriate parent trait of an op depends on the type of +operation: + +- `base` => All tensor kinds should implement these operations (Reshape, into_data, etc.). The + implementation is in `burn-tensor/src/tensor/api/base.rs`. +- `numeric` => All tensors that are numeric by nature should implement these operations (Add, Sub, + Div, etc.). The implementation is in `burn-tensor/src/tensor/api/numeric.rs`. +- `Float` => Tensor operations are only available for float tensors. The implementation is in + `burn-tensor/src/tensor/api/float.rs`. +- `Int` => Tensor operations are only available for int tensors. The implementation is in + `burn-tensor/src/tensor/api/int.rs`. +- `bool` => Tensor operations are only available for bool tensors. The implementation is in + `burn-tensor/src/tensor/api/bool.rs`. + +`Numeric` is directly implemented for `Float` and `Int` tensors, and in general, The implementations +for these methods are calling the corresponding `{Int|Float}TensorOp` method defined in the backend +supertrait. + +Anything that is implemented by numeric should have an implementation in the `{Int|Float}TensorOp` +traits, though it may be avoidable if the operation for one type requires casting to the other type. +To provide an example, Powf should be implemented for `Int` tensors, but it should not be an Int +Tensor Operation. The LHS should be converted to a float, and the output should be converted back to +an int. So it's possible to avoid implementing `IntTensorOp` altogether. + +additionally there are some operations that should be defined as functions instead of tensor/tensor +op methods. these are: + +`module` => These should be exported as functions instead of methods on tensors. The implementation +is in `burn-tensor/src/tensor/module.rs` (Might be moved to `tensor/api/module.rs`). `activation` => +These should also be exported as functions instead of methods on tensors. The implementation is in +`burn-tensor/src/tensor/activation/base.rs` (Might be moved to `tensor/api/activation.rs`). + +Note that some activations are just a combination of backend operations and are not declared in +`burn-tensor/src/tensor/ops/activation.rs` diff --git a/contributor-book/src/project-architecture/backend.md b/contributor-book/src/project-architecture/backend.md index 0679f95cde..ce80a246bb 100644 --- a/contributor-book/src/project-architecture/backend.md +++ b/contributor-book/src/project-architecture/backend.md @@ -1,4 +1,3 @@ - # Backend The Backend trait abstracts multiple things: @@ -13,23 +12,28 @@ The Backend trait abstracts multiple things: - Int tensor operations (kernels) - Bool tensor operations (kernels) -Even though having one type for tensors is convenient for the tensor API, it can be cumbersome when implementing a backend. -Therefore, backends can decide, through associated types, what types they want to use for their int, float, and bool tensors. -Since float and int can have multiple precisions, the float and int element types are also associated types that must be declared by the backend. +Even though having one type for tensors is convenient for the tensor API, it can be cumbersome when +implementing a backend. Therefore, backends can decide, through associated types, what types they +want to use for their int, float, and bool tensors. Since float and int can have multiple +precisions, the float and int element types are also associated types that must be declared by the +backend. -Note that the backend chooses the precision and not the user. -Since not all backends will support the same element types, no assumptions must be made. -Therefore, there are no methods on tensors to change the precision, except for the `to_full_precision` function, which ensures numerical stability on the current backend. -Backend implementations can provide a way to choose the precision, which can be accomplished with a generic parameter (e.g. `NdArray`). +Note that the backend chooses the precision and not the user. Since not all backends will support +the same element types, no assumptions must be made. Therefore, there are no methods on tensors to +change the precision, except for the `to_full_precision` function, which ensures numerical stability +on the current backend. Backend implementations can provide a way to choose the precision, which can +be accomplished with a generic parameter (e.g. `NdArray`). -To be as general as possible, tensor operations are implemented as plain functions. -There is no object or self, just functions that take tensors as input and often return tensors as output as well. -Backend implementations are free to use their own patterns to implement these kernels. -Note that Burn is a dynamic graph deep learning framework, so backends may have to implement asynchronous kernel executions for performance reasons. +To be as general as possible, tensor operations are implemented as plain functions. There is no +object or self, just functions that take tensors as input and often return tensors as output as +well. Backend implementations are free to use their own patterns to implement these kernels. Note +that Burn is a dynamic graph deep learning framework, so backends may have to implement asynchronous +kernel executions for performance reasons. ## Autodiff -As of now, there is only one backend decorator that supports autodiff. -It follows the decorator pattern, making any backend differentiable. -However, the `AutodiffBackend` trait abstracts how gradients are calculated, and other approaches to autodiff might be added later. -For more information about how the current autodiff backend works, you can read this [blog post](https://burn.dev/blog/burn-rusty-approach-to-tensor-handling). +As of now, there is only one backend decorator that supports autodiff. It follows the decorator +pattern, making any backend differentiable. However, the `AutodiffBackend` trait abstracts how +gradients are calculated, and other approaches to autodiff might be added later. For more +information about how the current autodiff backend works, you can read this +[blog post](https://burn.dev/blog/burn-rusty-approach-to-tensor-handling). diff --git a/contributor-book/src/project-architecture/module.md b/contributor-book/src/project-architecture/module.md index 5a7d16e3db..9d9783fb17 100644 --- a/contributor-book/src/project-architecture/module.md +++ b/contributor-book/src/project-architecture/module.md @@ -1,72 +1,93 @@ # Module -Modules are a way of creating neural network structures that can be easily optimized, saved, and loaded with little to no boilerplate. -Unlike other frameworks, a module does not force the declaration of the forward pass, leaving it up to the implementer to decide how it should be defined. +Modules are a way of creating neural network structures that can be easily optimized, saved, and +loaded with little to no boilerplate. Unlike other frameworks, a module does not force the +declaration of the forward pass, leaving it up to the implementer to decide how it should be +defined. -Additionally, most modules are created using a (de)serializable configuration, which defines the structure of the module and its hyper-parameters. -Parameters and hyper-parameters are not serialized into the same file and both are normally necessary to load a module for inference. +Additionally, most modules are created using a (de)serializable configuration, which defines the +structure of the module and its hyper-parameters. Parameters and hyper-parameters are not serialized +into the same file and both are normally necessary to load a module for inference. ## Optimization -Optimization is normally done with gradient descent (or ascent for reinforcement learning), and it is important to provide an easy API for optimizing modules. +Optimization is normally done with gradient descent (or ascent for reinforcement learning), and it +is important to provide an easy API for optimizing modules. ### Constraints -1. **Users should be able to control what is optimized.** - Modules can contain anything for maximum flexibility, but not everything needs to be optimized. -2. **Optimizers should have a serializable state that is updated during training.** - Many optimizers keep track of previous gradients to implement some form of momentum. - However, the state can be anything, not just tensors, allowing for easy implementation of any kind of optimizer. -3. **The learning rate can be updated during training.** - Learning rate schedulers are often used during training and should be considered as a key aspect. +1. **Users should be able to control what is optimized.** Modules can contain anything for maximum + flexibility, but not everything needs to be optimized. +2. **Optimizers should have a serializable state that is updated during training.** Many optimizers + keep track of previous gradients to implement some form of momentum. However, the state can be + anything, not just tensors, allowing for easy implementation of any kind of optimizer. +3. **The learning rate can be updated during training.** Learning rate schedulers are often used + during training and should be considered as a key aspect. ### Solution -`Module` trait defined in [`burn-core/src/module/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/module/base.rs#L83) -`Optimizer` trait defined in [`burn-core/src/optim/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/base.rs#L8) - -The solution to this problem comprises multiple parts. -Firstly, the `Optimizer` trait is quite similar to the `Module` trait, in terms of saving and loading the state. Please refer to the [serialization](./serialization.md) section for more details. - -Secondly, two traits were created. -The `Optimizer` trait is general and relatively unopinionated, with a simple `step` method that takes a learning rate, a module, and the gradients. -The other trait, `SimpleOptimizer`, aims to provide an easier API for implementing new optimizers. -The goal is to allow implementations to avoid handling missing gradients, loading and exporting records, navigating the module parameter structure, handling tracked and untracked tensors, and other such tasks. - -Thirdly, each tensor that will be optimized needs to be wrapped into a `Param` struct, which gives them an ID used for (de)serialization and to associate the state of the optimizer to each parameter. -The `Module` trait has two ways to navigate over parameters. -The first one is the `map` function, which returns `Self` and makes it easy to implement any transformation and mutate all parameters. -The second one is the `visit` function, which has a similar signature but does not mutate the parameter tensors. +`Module` trait defined in +[`burn-core/src/module/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/module/base.rs#L83) +`Optimizer` trait defined in +[`burn-core/src/optim/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/base.rs#L8) + +The solution to this problem comprises multiple parts. Firstly, the `Optimizer` trait is quite +similar to the `Module` trait, in terms of saving and loading the state. Please refer to the +[serialization](./serialization.md) section for more details. + +Secondly, two traits were created. The `Optimizer` trait is general and relatively unopinionated, +with a simple `step` method that takes a learning rate, a module, and the gradients. The other +trait, `SimpleOptimizer`, aims to provide an easier API for implementing new optimizers. The goal is +to allow implementations to avoid handling missing gradients, loading and exporting records, +navigating the module parameter structure, handling tracked and untracked tensors, and other such +tasks. + +Thirdly, each tensor that will be optimized needs to be wrapped into a `Param` struct, which gives +them an ID used for (de)serialization and to associate the state of the optimizer to each parameter. +The `Module` trait has two ways to navigate over parameters. The first one is the `map` function, +which returns `Self` and makes it easy to implement any transformation and mutate all parameters. +The second one is the `visit` function, which has a similar signature but does not mutate the +parameter tensors. #### SimpleOptimizer -located in [`burn-core/src/optim/simple/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/simple/base.rs#L9) +located in +[`burn-core/src/optim/simple/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/simple/base.rs#L9) The `SimpleOptimizer` has two major assumptions: -1. The state of the optimizer is linked to each parameter. - In other words, each parameter has its own optimizer state, decoupled from the other parameters. +1. The state of the optimizer is linked to each parameter. In other words, each parameter has its + own optimizer state, decoupled from the other parameters. 2. The state of the optimizer implements `Record`, `Clone`, and has a `'static` lifetime. -The benefits of those assumptions materialize in simplicity with little loss in flexibility. -The state associative type is also generic over the dimension, making it extremely easy to include tensors in the state that share the same dimensionality as its parameter. +The benefits of those assumptions materialize in simplicity with little loss in flexibility. The +state associative type is also generic over the dimension, making it extremely easy to include +tensors in the state that share the same dimensionality as its parameter. -To wrap a simple optimizer into the more general `Optimizer` trait, the `OptimizerAdaptor` struct is used. +To wrap a simple optimizer into the more general `Optimizer` trait, the `OptimizerAdaptor` struct is +used. #### OptimizerAdaptor -Located in in [`burn-core/src/optim/simple/adapter.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/simple/adaptor.rs#L14) +Located in in +[`burn-core/src/optim/simple/adapter.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/simple/adaptor.rs#L14) -The `OptimizerAdaptor` is a simple struct composed of a `SimpleOptimizer` and a hashmap with all records associated with each parameter ID. +The `OptimizerAdaptor` is a simple struct composed of a `SimpleOptimizer` and a hashmap with all +records associated with each parameter ID. When performing an optimization step, the adaptor handles the following: 1. Updates each parameter tensor in the given module using the `Module::map` function. 2. Checks if a gradient for the current tensor exists. -3. Makes sure that the gradient, the tensor, and the optimizer state associated with the current parameter are on the same device. - The device can be different if the state is loaded from disk to restart training. -4. Performs the simple optimizer step using the inner tensor since the operations done by the optimizer should not be tracked in the autodiff graph. -5. Updates the state for the current parameter and returns the updated tensor, making sure it's properly registered into the autodiff graph if gradients are marked as required. - -Note that a parameter can still be updated by another process, as is the case with running metrics used in batch norm. -These tensors are still wrapped using the `Param` struct so that they are included in the module's state and given a proper parameter ID, but they are not registered in the autodiff graph. +3. Makes sure that the gradient, the tensor, and the optimizer state associated with the current + parameter are on the same device. The device can be different if the state is loaded from disk to + restart training. +4. Performs the simple optimizer step using the inner tensor since the operations done by the + optimizer should not be tracked in the autodiff graph. +5. Updates the state for the current parameter and returns the updated tensor, making sure it's + properly registered into the autodiff graph if gradients are marked as required. + +Note that a parameter can still be updated by another process, as is the case with running metrics +used in batch norm. These tensors are still wrapped using the `Param` struct so that they are +included in the module's state and given a proper parameter ID, but they are not registered in the +autodiff graph. diff --git a/contributor-book/src/project-architecture/serialization.md b/contributor-book/src/project-architecture/serialization.md index 58390fa737..4be632936a 100644 --- a/contributor-book/src/project-architecture/serialization.md +++ b/contributor-book/src/project-architecture/serialization.md @@ -1,82 +1,101 @@ - # Serialization An important aspect of a deep learning framework is the ability to save and load models from disk. -Despite appearing as a simple feature, it involves numerous constraints that require a proper solution. +Despite appearing as a simple feature, it involves numerous constraints that require a proper +solution. ## Constraints -1. **Users should be able to declare the precision of the model to be saved, independent of the backend in use.** +1. **Users should be able to declare the precision of the model to be saved, independent of the + backend in use.** - The modules should not be duplicated in RAM in another precision to support this. - Conversion should be done lazily during (de)serialization. + The modules should not be duplicated in RAM in another precision to support this. Conversion + should be done lazily during (de)serialization. 2. **Users should be able to add any field to a module, even fields that are not serializable.** - This can include constants, database connections, other module references, or any other information. - Only parameters should be serialized since the structure of the module itself should be encapsulated with module configurations (hyper-parameters). + This can include constants, database connections, other module references, or any other + information. Only parameters should be serialized since the structure of the module itself should + be encapsulated with module configurations (hyper-parameters). 3. **Users should be able to declare the format in which the module should be saved.** - This can involve saving to a compressed JSON file or directly to bytes in memory for `no-std` environments. + This can involve saving to a compressed JSON file or directly to bytes in memory for `no-std` + environments. -4. **Users should be able to create a module with its saved parameters without having to initialize the module first.** +4. **Users should be able to create a module with its saved parameters without having to initialize + the module first.** - This will avoid unnecessary module initialization and tensor loading, resulting in reduced cold start when dealing with inference. + This will avoid unnecessary module initialization and tensor loading, resulting in reduced cold + start when dealing with inference. In addition to all of these constraints, the solution should be easy to use. ## Solution -In order to be able to add any field to a module without requiring it to be (de)serializable, we decouple the module type from its state. -We create a new type for each module that only contains the parameters that need to be saved. -To generate that type automatically, the user must either declare which field is a parameter or a constant, or we assume that each field implements the module trait. +In order to be able to add any field to a module without requiring it to be (de)serializable, we +decouple the module type from its state. We create a new type for each module that only contains the +parameters that need to be saved. To generate that type automatically, the user must either declare +which field is a parameter or a constant, or we assume that each field implements the module trait. -The second solution was chosen as it simplifies the code generation and reduces the size of the user API. -This means that the `Module` trait should be implemented by [primitives types](./burn-core/src/module/param/primitive.rs). -The following diagrams highlight the main types and traits used in the solution. +The second solution was chosen as it simplifies the code generation and reduces the size of the user +API. This means that the `Module` trait should be implemented by +[primitives types](./burn-core/src/module/param/primitive.rs). The following diagrams highlight the +main types and traits used in the solution.

Module Serialization Types

-The way the types interact with each other is pretty straightforward. -First, a module can be converted into a record using `into_record()`. -Note that tensors can be cloned, but it won't actually copy any data; it will create another reference to the same data. - -Then, a `Recorder` instance can be used to serialize any record. -The `Recorder` has the `PrecisionSettings` type as associate type, so any record will be serialized using the settings provided at the creation of the `Recorder` instance. -Note that tensors implement record, and their item is just a wrapper struct that contains information about the precision in which the tensor should be saved or loaded. -No actual copy of the tensor is made until this point. -The tensor is converted to the `Data` struct and then converted into the specified precision only when `serialize()` or `deserialize()` are called, which makes the whole process lazy. - -To recapitulate, the `Module` trait has an associated type that implements `Record`, which only contains the parameters of the model. -The `Record` trait has a generic associated type (GAT) that specifies a family of types that can be (de)serialized given any `PrecisionSettings`. -Records are therefore decoupled from the backend in use, and the saved items can be loaded on any backend with any precision, since the conversion is type-safe and done when `serialize()` and `deserialize()` are called. -All of the types are generated using simple derive macros without any conditional statements or complex syntax, as `Record` and `Module` are implemented for all primitive types. -This makes the code simple and easy to maintain. -In addition, you can extend the current system with your own `Recorder` and `PrecisionSettings` to control how your modules should be saved and loaded. +The way the types interact with each other is pretty straightforward. First, a module can be +converted into a record using `into_record()`. Note that tensors can be cloned, but it won't +actually copy any data; it will create another reference to the same data. + +Then, a `Recorder` instance can be used to serialize any record. The `Recorder` has the +`PrecisionSettings` type as associate type, so any record will be serialized using the settings +provided at the creation of the `Recorder` instance. Note that tensors implement record, and their +item is just a wrapper struct that contains information about the precision in which the tensor +should be saved or loaded. No actual copy of the tensor is made until this point. The tensor is +converted to the `Data` struct and then converted into the specified precision only when +`serialize()` or `deserialize()` are called, which makes the whole process lazy. + +To recapitulate, the `Module` trait has an associated type that implements `Record`, which only +contains the parameters of the model. The `Record` trait has a generic associated type (GAT) that +specifies a family of types that can be (de)serialized given any `PrecisionSettings`. Records are +therefore decoupled from the backend in use, and the saved items can be loaded on any backend with +any precision, since the conversion is type-safe and done when `serialize()` and `deserialize()` are +called. All of the types are generated using simple derive macros without any conditional statements +or complex syntax, as `Record` and `Module` are implemented for all primitive types. This makes the +code simple and easy to maintain. In addition, you can extend the current system with your own +`Recorder` and `PrecisionSettings` to control how your modules should be saved and loaded. ### Pros - All constraints are respected. -- The code is simple and easy to maintain, with very few conditional statements. - It is just recursive data structures, where all the complexity is handled by the framework in primitive implementations. -- The user API is simple and small, with only two derives (`Record` and `Module`) and no additional attributes. -- Users can create their own `Module` and `Record` primitive types, which gives them the flexibility to control how their data is serialized without having to fork the framework. +- The code is simple and easy to maintain, with very few conditional statements. It is just + recursive data structures, where all the complexity is handled by the framework in primitive + implementations. +- The user API is simple and small, with only two derives (`Record` and `Module`) and no additional + attributes. +- Users can create their own `Module` and `Record` primitive types, which gives them the flexibility + to control how their data is serialized without having to fork the framework. ### Cons -- There are more types, but most of them are automatically generated and single-purpose, so users don't need to interact with them for common use cases. - However, they can do so if necessary. -- When instantiating a new record manually, each field must be set to something, even if the type itself is `()`, which represents no value. - Since the code generation step uses associative types, it doesn't know that a field type is actually nothing. - Creating a record manually without using the generated function `into_record` or loading it from a file is only useful to load a set of parameters into a module from an arbitrary source. - Using the record may not be the optimal solution to this problem, and another API could be created in the future. +- There are more types, but most of them are automatically generated and single-purpose, so users + don't need to interact with them for common use cases. However, they can do so if necessary. +- When instantiating a new record manually, each field must be set to something, even if the type + itself is `()`, which represents no value. Since the code generation step uses associative types, + it doesn't know that a field type is actually nothing. Creating a record manually without using + the generated function `into_record` or loading it from a file is only useful to load a set of + parameters into a module from an arbitrary source. Using the record may not be the optimal + solution to this problem, and another API could be created in the future. ### Compatibility -Record may become incompatible with previous versions of Burn, depending on the chosen format. -The more compact format (bincode) store minimal information about the type, making it significantly smaller but less resilient to type changes such adding an optional field. -At some point, it might be necessary to provide a translation script that can translate a more resilient format from a previous version to a more compact one. \ No newline at end of file +Record may become incompatible with previous versions of Burn, depending on the chosen format. The +more compact format (bincode) store minimal information about the type, making it significantly +smaller but less resilient to type changes such adding an optional field. At some point, it might be +necessary to provide a translation script that can translate a more resilient format from a previous +version to a more compact one. From 20057bf3a1eb76a76293bcaaeafa2daefb33bca2 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Wed, 31 Jan 2024 14:49:44 -0600 Subject: [PATCH 23/32] implemented suggestions --- .../src/guides/adding-a-new-operation-to-burn.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index 9b7d4c2ea8..a82a48cd43 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -38,9 +38,7 @@ In this case, we only don't need to worry about `Bool` Tensors. Ops for `Float` [burn-tensor/src/tensor/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/ops/tensor.rs#L873), and for `Int` under [`burn-tensor/src/tensor/ops/int_tensor.rs`](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-tensor/src/tensor/ops/int_tensor.rs#L486). -The current convention is for int and bool ops, to be prefixed with `int` or `bool`, respectively. -As we need to use floating point powers to ensure precision, the `powi` functions are given a -default implementation that converts the right hand side of the operator to a float. +The current convention is ops of each type, if not unique to that type, are prefixed with the type. so `powf` and sundry would be defined as `int_powf` for `IntTensorOps` and `float_powf` for `FloatTensorOps`. If an op is unique to a type, then it should be implemented under `burn-tensor/src/api/{type}.rs`. For example, here is an implementation for [`sin` under `burn-tensor/src/api/float.rs`](https://github.com/tracel-ai/burn/blob/2acf6561dc9e173870d4209ed40ebbdcf7e3888c/burn-tensor/src/tensor/api/float.rs#L78) which obviously doesn't make sense for `Int` or `Bool` tensors. The `Int` Tensor function use the ones defined for Float with 2 extra cast (LHS to a `Float` tensor, Output to an `Int`). Given that the rest of the code will only look at the float implementations. @@ -98,8 +96,8 @@ plug in your operator in terms of \\(x\\) and \\(y\\), and just swap out the var ### Testing autodiff Test for autodiff go under -[burn-autodiff/src/tests/foo.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) -(replacing foo for whatever makes sense for your op), and for tensor operations both the left and +[burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) +(replacing `op_name` for whatever makes sense for your op), and for tensor operations both the left and right side need to be verified. The easiest way to do this, is to 1. use small tensors with simple values From f0ba1d2635c9b5e362b2e7f99f981b833b86a067 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Wed, 31 Jan 2024 14:53:44 -0600 Subject: [PATCH 24/32] implemented suggestion --- .../src/getting-started/setting-up-the-environment.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index 5975a7c857..f5a42191de 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -29,7 +29,7 @@ following command[^1]: cargo install mdbook ``` -Also instead of running `./run_checks.sh all`, you can run `./run_checks.sh typo` to only check for +Also instead of running `./run_checks.sh all`, you can run `./run_checks.sh typo`, or `cargo xtasks run-checks typo`, to only check for misspellings. This will install [typo](https://crates.io/crates/typos-cli), and if any are encountered you should be able to run `typo -w /path/to/book` to fix them. From 3ea32192a431bfec33a9671fe3c638f274c2c31b Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Wed, 31 Jan 2024 19:43:32 -0600 Subject: [PATCH 25/32] updated testing autodiff section --- burn-tensor/src/tests/ops/powf.rs | 1 - contributor-book/src/guides/adding-a-new-operation-to-burn.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/burn-tensor/src/tests/ops/powf.rs b/burn-tensor/src/tests/ops/powf.rs index 7a9c05bcee..90c2c1b9fb 100644 --- a/burn-tensor/src/tests/ops/powf.rs +++ b/burn-tensor/src/tests/ops/powf.rs @@ -29,7 +29,6 @@ mod tests { #[test] fn should_support_neg_values_with_even_power() { - let data = Data::from([[1.0, -1.0, -2.0], [-3.0, -4.0, -5.0]]); let tensor = Tensor::::from_data(data, &Default::default()); let pow = Data::from([[2.0, 2.0, 4.0], [4.0, 4.0, 2.0]]); diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index a82a48cd43..1dbe2e155c 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -101,7 +101,7 @@ Test for autodiff go under right side need to be verified. The easiest way to do this, is to 1. use small tensors with simple values -2. pop open a terminal and launch `ipython` import `numpy` (or just use +2. Compute the expected results for the chosen tensors, using some independant and reliable tool. For instance, you can pop open a terminal and launch `ipython` import `numpy` (or just use [google colab](https://colab.google/) if you don't have the packages installed and don't want to install them), and do the calculations by hand. 3. comparing the actual to expected output for lhs, rhs and regular operation From 2739781dba4d37bd34ca95bfe3854ba4ce2a1115 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Wed, 31 Jan 2024 19:45:10 -0600 Subject: [PATCH 26/32] fixed typo --- contributor-book/src/guides/adding-a-new-operation-to-burn.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index 1dbe2e155c..1f2205a073 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -101,7 +101,7 @@ Test for autodiff go under right side need to be verified. The easiest way to do this, is to 1. use small tensors with simple values -2. Compute the expected results for the chosen tensors, using some independant and reliable tool. For instance, you can pop open a terminal and launch `ipython` import `numpy` (or just use +2. Compute the expected results for the chosen tensors, using some independent and reliable tool. For instance, you can pop open a terminal and launch `ipython` import `numpy` (or just use [google colab](https://colab.google/) if you don't have the packages installed and don't want to install them), and do the calculations by hand. 3. comparing the actual to expected output for lhs, rhs and regular operation From 5d06209ff8b516442067e2b0174fe5294c75bcd6 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 2 Feb 2024 08:39:49 -0600 Subject: [PATCH 27/32] Apply suggestions from code review Co-authored-by: Sylvain Benner --- .../issues-while-adding-ops.md | 2 +- .../configuring-your-editor.md | 2 +- .../src/getting-started/testing.md | 6 +-- contributor-book/src/guides/README.md | 2 +- .../guides/adding-a-new-operation-to-burn.md | 44 +++++++++---------- contributor-book/src/overview.md | 6 +-- .../src/project-architecture/Tensor.md | 2 +- .../src/project-architecture/module.md | 4 +- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md index a5181e8e88..fd9e3c5248 100644 --- a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md +++ b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md @@ -28,7 +28,7 @@ error[E0599]: no method named `powi` found for struct `Tensor` in the current sc For more information about an error, try `rustc --explain E0308`. error: could not compile `onnx-tests` (test "onnx_tests") due to 3 previous errors ``` -So if you are getting this, you probably didn't impl your operator for the actual Tensor struct. +If you are getting this error, you probably didn't implement your operator for the actual Tensor struct. This issue was encountered when adding the Pow operator. The operation was added to the `FloatTensorOps` and `IntTensorOp` traits, but not for the numeric trait (under `burn-tensor/src/tensor/api/numeric.rs`). This, coupled with `powf` existing prior to the PR though diff --git a/contributor-book/src/getting-started/configuring-your-editor.md b/contributor-book/src/getting-started/configuring-your-editor.md index 5d3657ce3a..62f8738719 100644 --- a/contributor-book/src/getting-started/configuring-your-editor.md +++ b/contributor-book/src/getting-started/configuring-your-editor.md @@ -12,7 +12,7 @@ haven't already done it. - [serayuzgur.crates](https://marketplace.visualstudio.com/items?itemName=serayuzgur.crates) - [vadimcn.vscode-lldb](https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb) -2. Open `Command Palette` with Ctrl+Shift+P or F1 and type +2. Open `Command Palette` with `Ctrl+Shift+P` or `F1` and type `LLDB: Generate Launch Configurations from Cargo.toml` then select it, this will generate a file that should be saved as `.vscode/launch.json`. diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index 6d8ba5d702..2aa8a7bc26 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -7,7 +7,7 @@ The following examples use matrix multiplication operation. Test for Tensor operations (as in given this input, expect it match or approximate this output) are defined only in [`burn-tensor/src/test/ops`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/ops/matmul.rs#L1) -and not in the backends, with the exception of `burn-autodiff`. These test are added to the +and not in the backends, with the exception of `burn-autodiff`. These tests are added to the `testgen_all` macro rule in [`burn-tensor/src/test/mod.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-tensor/src/tests/mod.rs#L59). This is then propagated to the existing backends without any additional work. @@ -18,7 +18,7 @@ The following examples use the power operation. Tests for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) -(replace `foo` for whatever makes sense for your op), and for tensor operations both the left and +(replace `op_name` with whatever makes sense for your op), and for tensor operations both the left and right sides need to be verified. The easiest way to do this, is to: 1. use small tensors with simple values @@ -27,6 +27,6 @@ right sides need to be verified. The easiest way to do this, is to: install the packages on your system. 3. compare the actual output to the expected output for lhs, rhs and regular operation -generally, it seems preferable to use +Generally, it seems preferable to use `actual_output_tensor.into_data().assert_approx_eq(&expected_tensor_data,3)` to `assert_eq!(...` due to occasional hiccups with floating point calculations. diff --git a/contributor-book/src/guides/README.md b/contributor-book/src/guides/README.md index aad15f3626..295a1f28dc 100644 --- a/contributor-book/src/guides/README.md +++ b/contributor-book/src/guides/README.md @@ -1,3 +1,3 @@ # Guides for Contributors -The following guides are meant to help contributors trying to accomplish specific tasks, such as adding a new operations to Burn or generating test models for burn-import. \ No newline at end of file +The following guides are meant to help contributors trying to accomplish specific tasks, such as adding new operations to Burn or generating test models for `burn-import`. \ No newline at end of file diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index 1f2205a073..f64ce39b69 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -1,7 +1,7 @@ # Adding a new operation to burn -Let's discuss how one might go about adding new operators to Burn, using the example of the recently -added [pow operator](https://github.com/tracel-ai/burn/pull/1133/files) (as of 01/24/2024). In that +Let's discuss how one might go about adding new operators to Burn, using the example of the +pow operator added in [this PR](https://github.com/tracel-ai/burn/pull/1133/files). In that PR, the following things took place (albeit not in this order) ## Adding the Op to burn-tensor @@ -13,34 +13,34 @@ home of all tensor operations that are numeric in nature and that are shared by Tensor types. More information on the relationship between Tensor modules can be found under the section for [Tensor Architecture](../project-architecture/Tensor.md#tensorops). -here is where pow was added to `burn-tensor/src/tensor/api/numeric.rs`: +Here is where pow was added to `burn-tensor/src/tensor/api/numeric.rs`: 1. for the [`Tensor` struct](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L553) 2. for the [numeric trait](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1618) -3. for the impl of numeric for +3. for the implementation of numeric for [float](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L2186) and [int](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/numeric.rs#L1903) -Tensor is a struct that has a single member: `primitive`(defined -[here](https://github.com/tracel-ai/burn/blob/main/burn-tensor/src/tensor/api/base.rs)), That is +Tensor is a struct that has a single member: `primitive` (defined +[here](https://github.com/tracel-ai/burn/blob/main/burn-tensor/src/tensor/api/base.rs)), that is defined by it's [`Kind`](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/api/kind.rs#L15): one of `Bool`, `Float`, or `Int` (those linked in 3). These call the ops for that data type defined in the [`Backend`](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/backend/base.rs#L52) -supertrait[^1]. This is the trait that is then implemented by the different `burn-` back-ends (such +supertrait[^1]. This is the trait that is then implemented by the different `burn-` backends (such as `burn-ndarray` and `burn-wgpu`) which implement the functions if no default is provided. -In this case, we only don't need to worry about `Bool` Tensors. Ops for `Float` is implemented under +In this case, we only need to worry about `Bool` Tensors. Ops for `Float` is implemented under [burn-tensor/src/tensor/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/ops/tensor.rs#L873), and for `Int` under [`burn-tensor/src/tensor/ops/int_tensor.rs`](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-tensor/src/tensor/ops/int_tensor.rs#L486). -The current convention is ops of each type, if not unique to that type, are prefixed with the type. so `powf` and sundry would be defined as `int_powf` for `IntTensorOps` and `float_powf` for `FloatTensorOps`. If an op is unique to a type, then it should be implemented under `burn-tensor/src/api/{type}.rs`. For example, here is an implementation for [`sin` under `burn-tensor/src/api/float.rs`](https://github.com/tracel-ai/burn/blob/2acf6561dc9e173870d4209ed40ebbdcf7e3888c/burn-tensor/src/tensor/api/float.rs#L78) which obviously doesn't make sense for `Int` or `Bool` tensors. +The current convention is ops of each type, if not unique to that type, are prefixed with the type. So `powf` and sundry would be defined as `int_powf` for `IntTensorOps` and `float_powf` for `FloatTensorOps`. If an op is unique to a type, then it should be implemented under `burn-tensor/src/api/{type}.rs`. For example, here is an implementation for [`sin` under `burn-tensor/src/api/float.rs`](https://github.com/tracel-ai/burn/blob/2acf6561dc9e173870d4209ed40ebbdcf7e3888c/burn-tensor/src/tensor/api/float.rs#L78) which obviously doesn't make sense for `Int` or `Bool` tensors. -The `Int` Tensor function use the ones defined for Float with 2 extra cast (LHS to a `Float` tensor, +The `Int` Tensor function uses the ones defined for Float with 2 extra casts (LHS to a `Float` tensor, Output to an `Int`). Given that the rest of the code will only look at the float implementations. ### Adding Test @@ -67,12 +67,12 @@ and need to: 3. Then defines what happens when a specific operation is tracked or untracked, where untracked just calls the function in the normal way, and tracked executes the backward function defined above -steps 1 and 3 are boilerplate, so much so that you can probably just copy the contents of another op +Steps 1 and 3 are boilerplate, so much so that you can probably just copy the contents of another op of the same type (binary, unary) and change the name of the struct, and ensure that either both -sides have the data they need (if they need to have a copy of the opposite sided tensor, clone it's +sides have the data they need (if they need to have a copy of the opposite sided tensor, clone its contents). -now for step 2. Since a significant number of the people reading this probably haven't touched +Now for step 2. Since a significant number of the people reading this probably haven't touched calculus either ever, or since however long ago you took the appropriate course, I'll assume that you, the observer, have some prior knowledge of calculus but would benefit from a review of the concepts. If this is not the case, I apologize, you can probably skim this section. @@ -82,7 +82,7 @@ derivatives with respect to the left and right sided tensors. Let's define the operator as a function \\(f(x,y)=x^{y}\\) , where \\(x\\) is the left hand tensor and \\(y\\) is the right handed tensor. The two closures are defining the partial derivatives of -\\(f\\) with respect to \\(x\\),\\(y\\). The eli5 is treat the other variable as a constant +\\(f\\) with respect to \\(x\\),\\(y\\). Treat the other variables as a constant $$\frac{\delta }{\delta x} (x^{y})= y \cdot x^{y-1}$$ is the left handed closure, and @@ -97,7 +97,7 @@ plug in your operator in terms of \\(x\\) and \\(y\\), and just swap out the var Test for autodiff go under [burn-autodiff/src/tests/{op_name}.rs](https://github.com/tracel-ai/burn/blob/4ca3e31601228952bb1c1492bc9cd2adf15b5cf1/burn-autodiff/src/tests/pow.rs#L31) -(replacing `op_name` for whatever makes sense for your op), and for tensor operations both the left and +(replacing `op_name` with whatever makes sense for your op), and for tensor operations both the left and right side need to be verified. The easiest way to do this, is to 1. use small tensors with simple values @@ -112,7 +112,7 @@ to occasional hiccups with floating point calculations. ## Adding the Op to other backends -most of these are fairly straightforward implementations. for reference here's pow's float +Most of these are fairly straightforward implementations. For reference here's pow's float implementation for torch, ndarray and candle backends: 1. Torch implementation in @@ -138,7 +138,7 @@ is currently only supported for `burn-wgpu`), so adding the operator won't invol calculation, you'll just be describing how the generated code should look. Most of this can be copy/pasted/adjusted from other functions. -here's how powf was added to burn fusion: +Here's how powf was added to burn fusion: 1. added powf to the float ops under [`burn-fusion/src/ops/float.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/ops/float.rs#L1758) @@ -151,7 +151,7 @@ Adding pow to wgpu was actually pretty easy due to the design. Element-wise tens vectorized scalar ops, and given that raising a tensor to a scalar power prior to the tensor version, I just reused the code for scalar powf. -here is where code was added +Here is where code was added 1. to the implementation of [`TensorOps` under `burn-wgpu/src/ops/float_ops.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/float_ops.rs#L513) @@ -173,14 +173,14 @@ other places within `burn-wgpu` where additions would have been necessary otherw ## Adding the Op to burn-import -I won't comment on generating the test onnx files or the test, as that is already covered +I won't comment on generating the ONNX test files or the tests, as this is already covered [in the ONNX to burn guide](onnx-to-burn-conversion-tool.md#adding-new-operators), this is more about the specific changes you need to make when adding new operators after you have generated the tests. The crate is divided into two sections `src/burn` and `src/onnx`. The code under the former corresponds to the operation you've implemented earlier in this guide, and the latter to the -operations defined in the onnx specification. So when you are loading a model, the operator is first +operations defined in the ONNX specification. So when you are loading a model, the operator is first parsed to an intermediate representation defined by `src/onnx`, and then mapped to a Burn operations defined under `src/burn/node`. @@ -193,9 +193,9 @@ Let's review the changes made for pow starting from `src/burn` and moving to `sr [`to_str` definition](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/burn/node/binary.rs#L9) 2. add an arm to the match statement inside the `into_burn` function in [burn-import/src/onnx/to_burn.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/to_burn.rs#L269) - for the onnx `NodeType`(corresponds to an op in the Onnx spec), and make a + for the ONNX `NodeType` (which corresponds to an op in the ONNX spec), and make a [`foo_conversion` function](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/to_burn.rs#L667) - that maps the onnx node to the binary type + that maps the ONNX node to the binary type 3. specify how dimensions for the output should be derived in [burn-import/src/onnx/dim_inference.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-import/src/onnx/dim_inference.rs#L103) diff --git a/contributor-book/src/overview.md b/contributor-book/src/overview.md index 8edff02a15..e7720e020b 100644 --- a/contributor-book/src/overview.md +++ b/contributor-book/src/overview.md @@ -3,7 +3,7 @@ Welcome to The Burn Contributor's Book 👋 This book will help you get acquainted with the internals of the Burn deep learning framework and -provide some guidance on how to contribute to the project. +provide some detailed guidance on how to contribute to the project. We have crafted some sections for you: @@ -11,7 +11,7 @@ We have crafted some sections for you: fundamentals, guiding you through tasks like setting up the development environment, how to run tests, and what you should check prior to each commit. - [Project Architecture](./project-architecture): This section will give you a more in-depth look at - the architecture of burn + the architecture of Burn - [Guides](./guides): We'll provide some guides on how to do specific tasks, such as adding a new operation to Burn. @@ -20,6 +20,6 @@ We have crafted some sections for you: that has you stumped, this is the section to check out prior to asking on the discord. It's a collection of errors encountered by contributors, what caused them, and how they were resolved. -As this book is geared more towards contributors rather than users of burn, we'll assume you have a +As this book is geared towards contributors and not towards users of Burn, we'll assume you have a good understanding of software development, but will make efforts to explain anything outside of that scope, or at least provide links to resources that explain it better than we can. diff --git a/contributor-book/src/project-architecture/Tensor.md b/contributor-book/src/project-architecture/Tensor.md index 3913e241b8..bbbefeef2a 100644 --- a/contributor-book/src/project-architecture/Tensor.md +++ b/contributor-book/src/project-architecture/Tensor.md @@ -50,7 +50,7 @@ To provide an example, Powf should be implemented for `Int` tensors, but it shou Tensor Operation. The LHS should be converted to a float, and the output should be converted back to an int. So it's possible to avoid implementing `IntTensorOp` altogether. -additionally there are some operations that should be defined as functions instead of tensor/tensor +Additionally there are some operations that should be defined as functions instead of tensor/tensor op methods. these are: `module` => These should be exported as functions instead of methods on tensors. The implementation diff --git a/contributor-book/src/project-architecture/module.md b/contributor-book/src/project-architecture/module.md index 9d9783fb17..23229db34f 100644 --- a/contributor-book/src/project-architecture/module.md +++ b/contributor-book/src/project-architecture/module.md @@ -51,7 +51,7 @@ parameter tensors. #### SimpleOptimizer -located in +Located in [`burn-core/src/optim/simple/base.rs`](https://github.com/tracel-ai/burn/blob/b9bd42959b0d3e755a25e383cb5b38beb25559b8/burn-core/src/optim/simple/base.rs#L9) The `SimpleOptimizer` has two major assumptions: @@ -87,7 +87,7 @@ When performing an optimization step, the adaptor handles the following: 5. Updates the state for the current parameter and returns the updated tensor, making sure it's properly registered into the autodiff graph if gradients are marked as required. -Note that a parameter can still be updated by another process, as is the case with running metrics +Note that a parameter can still be updated by another process, as it is the case with running metrics used in batch norm. These tensors are still wrapped using the `Param` struct so that they are included in the module's state and given a proper parameter ID, but they are not registered in the autodiff graph. From 726d2323a9d94808e437ec29a364f577719a395a Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 2 Feb 2024 11:09:19 -0600 Subject: [PATCH 28/32] fixes related to code review --- .../issues-while-adding-ops.md | 5 +++-- .../src/getting-started/setting-up-the-environment.md | 7 +++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md index fd9e3c5248..2dc1199cdc 100644 --- a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md +++ b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md @@ -12,8 +12,9 @@ it to this list. tests::maxmin::tests::test_mean_dim_2d stdout ---- thread 'tests::maxmin::tests::test_mean_dim_2d' panicked at burn-wgpu/src/lib.rs:49:5: assertion `left == right` failed left: Data { value: [1.0, 4.0], shape: Shape { dims: [2, 1] } } right: Data { value: [0.99999994, 3.9999998], shape: Shape { dims: [2, 1] } } ``` -If you encounter this, swap out the `assert_eq!` in the failing test for `tensor1.assert_approx_eq` -with `3` as the second argument. +If you encounter this, swap out the `assert_eq!` in the failing test for `tensor1.assert_approx_eq` with `3` as the second argument. The second arguments specifies the level of precision. `3` is equivalent to a less than 0.001 difference between the elements of the two tensors. + + ## Mismatched types and missing functions diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index f5a42191de..aef6552505 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -17,12 +17,15 @@ There are a few commands you want to run prior to any commit for a non-draft PR: ## Updating the burn semver version -To bump for the next version, edit the semantic version number in `burn/Cargo.toml`, and then run +If for some reason you need to bump for the next version (though that should probably be left to the maintainers), edit the semantic version number in `burn/Cargo.toml`, and then run `cargo update` to update the lock file. ## Contributing to either the Burn Book or Contributor Book -Both the Burn Book and the Contributor Book are built with mdbook. To install mdbook, run the + +Both the Burn Book and the Contributor Book are built with mdbook. If in the process of adding or modifying a page in the books, if you need to inspect the generated output(such as when using mathjax which seems prone to breakage), run use `mdbook --open ` or run `cargo xtask books {burn|contributor} open` which will install and use mdbook automatically. + +Alternatively, if you want to install mdbook directly, run the following command[^1]: ```bash From e9ee469d7d29a2d15c73f0e9bbefecd0ecaa62c0 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 2 Feb 2024 11:15:49 -0600 Subject: [PATCH 29/32] added links to discord server --- .../frequently-encountered-issues/issues-while-adding-ops.md | 2 +- contributor-book/src/getting-started/ReadMe.md | 2 +- contributor-book/src/overview.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md index 2dc1199cdc..ce43ba3cc9 100644 --- a/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md +++ b/contributor-book/src/frequently-encountered-issues/issues-while-adding-ops.md @@ -2,7 +2,7 @@ Below are some of the issues that were encountered while adding ops to the project. If you encounter an issue while adding an op that isn't listed here, and it's not obvious how to fix it, please add -it to this list. +it to this list. Also, reach out on the [discord server](https://discord.gg/uPEBbYYDB6) if you need help. ## Off by .000001 errors diff --git a/contributor-book/src/getting-started/ReadMe.md b/contributor-book/src/getting-started/ReadMe.md index 7e97955eb4..5bce636ebf 100644 --- a/contributor-book/src/getting-started/ReadMe.md +++ b/contributor-book/src/getting-started/ReadMe.md @@ -1,3 +1,3 @@ # Getting Started -This section is for setting up the environment and how to do basic development tasks such as running tests and checking your code before committing. \ No newline at end of file +This section is for setting up the environment and how to do basic development tasks such as running tests and checking your code before committing. If you need help with the process or run into issues, feel free to ask in the [discord server](https://discord.gg/uPEBbYYDB6) \ No newline at end of file diff --git a/contributor-book/src/overview.md b/contributor-book/src/overview.md index e7720e020b..f9c3ac9357 100644 --- a/contributor-book/src/overview.md +++ b/contributor-book/src/overview.md @@ -17,7 +17,7 @@ We have crafted some sections for you: operation to Burn. - [Frequently Encountered Issues](./frequently-encountered-issues): If you are running into an issue - that has you stumped, this is the section to check out prior to asking on the discord. It's a + that has you stumped, this is the section to check out prior to asking on the [discord](https://discord.gg/uPEBbYYDB6). It's a collection of errors encountered by contributors, what caused them, and how they were resolved. As this book is geared towards contributors and not towards users of Burn, we'll assume you have a From c1f63043250b2186519040af237161b0bf800831 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 2 Feb 2024 17:18:37 -0600 Subject: [PATCH 30/32] fixed a typo in the book xtask and implemented changes from code review --- .../getting-started/configuring-your-editor.md | 15 +++++++-------- .../getting-started/setting-up-the-environment.md | 14 ++++---------- xtask/src/books.rs | 2 +- 3 files changed, 12 insertions(+), 19 deletions(-) diff --git a/contributor-book/src/getting-started/configuring-your-editor.md b/contributor-book/src/getting-started/configuring-your-editor.md index 62f8738719..51d55de7e9 100644 --- a/contributor-book/src/getting-started/configuring-your-editor.md +++ b/contributor-book/src/getting-started/configuring-your-editor.md @@ -5,25 +5,24 @@ haven't already done it. ## VSCode -1. Install the following extensions: +Install the following extensions: - [rust-lang.rust-analyzer](https://marketplace.visualstudio.com/items?itemName=rust-lang.rust-analyzer) - [tamasfe.even-better-toml](https://marketplace.visualstudio.com/items?itemName=tamasfe.even-better-toml) - [serayuzgur.crates](https://marketplace.visualstudio.com/items?itemName=serayuzgur.crates) - [vadimcn.vscode-lldb](https://marketplace.visualstudio.com/items?itemName=vadimcn.vscode-lldb) -2. Open `Command Palette` with `Ctrl+Shift+P` or `F1` and type - `LLDB: Generate Launch Configurations from Cargo.toml` then select it, this will generate a file - that should be saved as `.vscode/launch.json`. +### Setting up the Debugger -3. Now you can enable breakpoint on code through IDE and then start debugging the library/binary you - want, such as the following example: +To use the debugger, follow these steps: +1. Open `Command Palette` with `Ctrl+Shift+P` or `F1` and type `LLDB: Generate Launch Configurations from Cargo.toml` then select it, this will generate a file that should be saved as `.vscode/launch.json`. +2. Select the configuration from the "run and debug" side panel (it have a infested play button), then select the target from +3. Now you can enable breakpoint on code through IDE and then start debugging the library/binary you want, such as the following example:
-4. If you're creating a new library or binary, keep in mind to repeat the step 2 to always keep a - fresh list of targets. +If you're creating a new library or binary, keep in mind to repeat the step 1. to always keep a fresh list of targets. ## Have another editor? Open a PR! diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index aef6552505..9f198377f6 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -13,7 +13,7 @@ There are a few commands you want to run prior to any commit for a non-draft PR: 2. `cargo fmt --all`, this will run rustfmt on all files in the project 3. `./run_checks.sh all`, this is a script located in the project root that builds and tests the project. It is required that this passes prior to merging a PR. Fair warning, running these tests - can take a while[^2]. + can take a while[^linux_mem_note]. ## Updating the burn semver version @@ -26,7 +26,7 @@ If for some reason you need to bump for the next version (though that should pro Both the Burn Book and the Contributor Book are built with mdbook. If in the process of adding or modifying a page in the books, if you need to inspect the generated output(such as when using mathjax which seems prone to breakage), run use `mdbook --open ` or run `cargo xtask books {burn|contributor} open` which will install and use mdbook automatically. Alternatively, if you want to install mdbook directly, run the -following command[^1]: +following command[^update_note]: ```bash cargo install mdbook @@ -36,12 +36,6 @@ Also instead of running `./run_checks.sh all`, you can run `./run_checks.sh typo misspellings. This will install [typo](https://crates.io/crates/typos-cli), and if any are encountered you should be able to run `typo -w /path/to/book` to fix them. -[^1]: - You might also want to install [cargo-update](https://github.com/nabijaczleweli/cargo-update) to - easily keep your tools up to date, though it is in no way required. +[^linux_mem_note]: If your system is running into issues with memory and you are on linux, you may want to switch to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console (and log in), and either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. -[^2]: - If your system is running into issues with memory and you are on linux, you may want to switch - to a [virtual console](https://wiki.archlinux.org/title/Linux_console#Virtual_consoles) to run - the tests. To do this, press `ctrl+alt+f3` to switch to a virtual console (and log in), and - either `ctrl+alt+f2` or `ctrl+alt+f1` to switch back to your graphical session. +[^update_note]: You might also want to install [cargo-update](https://github.com/nabijaczleweli/cargo-update) to easily keep your tools up to date, though it is in no way required. diff --git a/xtask/src/books.rs b/xtask/src/books.rs index fcaa00cc4f..f691bde699 100644 --- a/xtask/src/books.rs +++ b/xtask/src/books.rs @@ -72,7 +72,7 @@ impl Book { const BURN_BOOK_PATH: &'static str = "./burn-book"; const CONTRIBUTOR_BOOK_NAME: &'static str = "Contributor Book"; - const CONTRIBUTOR_BOOK_PATH: &'static str = "./burn-book"; + const CONTRIBUTOR_BOOK_PATH: &'static str = "./contributor-book"; pub(crate) fn run(book_arg: &BookKind) -> anyhow::Result<()> { let (book, command) = match book_arg { From b3e84ef84093c0f8b3aeea70c4845e7894744ae3 Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 2 Feb 2024 17:48:18 -0600 Subject: [PATCH 31/32] implemented suggestions from code reviews --- .../guides/adding-a-new-operation-to-burn.md | 45 +++++++------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/contributor-book/src/guides/adding-a-new-operation-to-burn.md b/contributor-book/src/guides/adding-a-new-operation-to-burn.md index f64ce39b69..63d113118e 100644 --- a/contributor-book/src/guides/adding-a-new-operation-to-burn.md +++ b/contributor-book/src/guides/adding-a-new-operation-to-burn.md @@ -31,10 +31,10 @@ defined by it's one of `Bool`, `Float`, or `Int` (those linked in 3). These call the ops for that data type defined in the [`Backend`](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/backend/base.rs#L52) -supertrait[^1]. This is the trait that is then implemented by the different `burn-` backends (such +supertrait[^supertrait]. This is the trait that is then implemented by the different `burn-` backends (such as `burn-ndarray` and `burn-wgpu`) which implement the functions if no default is provided. -In this case, we only need to worry about `Bool` Tensors. Ops for `Float` is implemented under +In this case, we don't need to worry about `Bool` Tensors. Ops for `Float` is implemented under [burn-tensor/src/tensor/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/3b7d9feede702cd616c273fa9eba9fbf14f66964/burn-tensor/src/tensor/ops/tensor.rs#L873), and for `Int` under [`burn-tensor/src/tensor/ops/int_tensor.rs`](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-tensor/src/tensor/ops/int_tensor.rs#L486). @@ -48,19 +48,18 @@ Output to an `Int`). Given that the rest of the code will only look at the float Additional Test should be added to `burn-tensor` under [`burn-tensor/src/tests/ops/{op_name}.rs`](https://github.com/tracel-ai/burn/burn-tensor/src/tests/ops/powf.rs), inserting the module name into `burn-tensor/src/tests/ops/mod.rs`. Then add it to the `testgen_all` -macro under `burn-tensor/src/tests/mod.rs`. This test is automatically added to the backends so it -isn't necessary to add them there, save for those that require specific testing such +macro under `burn-tensor/src/tests/mod.rs`. This macro is called from the `lib.rs` file in each backend, which autogenerates the tests for that specific backend. It isn't necessary to define tests in the backends directly, save for those that require specific testing such as`burn-autodiff` ## Adding the Op to the burn-autodiff Since this is probably the hardest and the least straightforward, we'll cover this backend -separately. Burn-autodiff enables other backends to use autodifferentiation[^2]. Ops for float types +separately. Burn-autodiff enables other backends to use autodifferentiation[^autodiff]. Ops for float types are implemented in [burn-autodiff/src/ops/tensor.rs](https://github.com/tracel-ai/burn/blob/e1d873abe2c2fa0bb316719c4761eaf796291166/burn-autodiff/src/ops/tensor.rs#L1523) and need to: -1. define a unit struct [^3] that implements a backward (pass) function +1. define a unit struct [^absolute_units] that implements a backward (pass) function 2. Within the backward function, as this is an elementwise binary operation it implements the binary function (from backward.rs under the same directory), the last 2 arguments are two closures that define the left and right partial derivatives. @@ -72,12 +71,10 @@ of the same type (binary, unary) and change the name of the struct, and ensure t sides have the data they need (if they need to have a copy of the opposite sided tensor, clone its contents). -Now for step 2. Since a significant number of the people reading this probably haven't touched -calculus either ever, or since however long ago you took the appropriate course, I'll assume that -you, the observer, have some prior knowledge of calculus but would benefit from a review of the -concepts. If this is not the case, I apologize, you can probably skim this section. +For those that need it, here is a quick refresher on the necessary calculus. If you +are familiar with how to calculate partial derivatives, you can skip this section. -In the case of pow, since this is a binary operation, the left and right functions are the partial +Since pow is a binary operation, the left and right functions are the partial derivatives with respect to the left and right sided tensors. Let's define the operator as a function \\(f(x,y)=x^{y}\\) , where \\(x\\) is the left hand tensor @@ -147,12 +144,9 @@ Here's how powf was added to burn fusion: 3. added powf to the implementations of `FloatOperationDescription` enum under [burn-fusion/src/stream/context.rs](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-fusion/src/stream/context.rs#L726) -Adding pow to wgpu was actually pretty easy due to the design. Element-wise tensor ops are just -vectorized scalar ops, and given that raising a tensor to a scalar power prior to the tensor -version, I just reused the code for scalar powf. - -Here is where code was added +Adding pow to wgpu was actually pretty easy due to the design. The way wgpu handles tensor-scalar operations is by transforming both into a sequence of vectorized scalar operations. Since powf already existed in burn-wgpu, It was pretty easy to reuse the existing implementation for the situation where both sides of the operation were tensors. +Here is where code was added for powf in burn-wgpu: 1. to the implementation of [`TensorOps` under `burn-wgpu/src/ops/float_ops.rs`](https://github.com/tracel-ai/burn/blob/0368409eb3a7beaeda598c0c8ce1dc0c2c8c07cc/burn-wgpu/src/ops/float_ops.rs#L513) 2. the function being called was added to @@ -162,14 +156,11 @@ Here is where code was added 4. A custom function generator was added to [`burn-wgpu/src/codegen/function.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/codegen/function.rs#L99) -Much of the logic for powf had already been defined, so not much needed to be added. The reason for -the custom powf function is generated, rather than directly using the underlying function as is the -case with other operations, is due to issues with case handling of the wgsl pow function, like 0 to +We needed to generate some custom wgsl code for powf, primarily due to issues with proper case handling of the wgsl pow function, like 0 to the 0 power being 1, and any negative number to an even power being positive. We reused as much as the existing logic as possible, such as the operation output description generation in [`burn-wgpu/src/fusion/elemwise/builder.rs`](https://github.com/tracel-ai/burn/blob/main/burn-wgpu/src/fusion/elemwise/optimization.rs) -and then branched at the last point based off the var type of the rhs. I don't know if there are -other places within `burn-wgpu` where additions would have been necessary otherwise. +and then branched at the last point based off the var type of the rhs. ## Adding the Op to burn-import @@ -203,14 +194,8 @@ And you're done! Congrats, you just fully added a new op to burn, and we are all the answer to [are we learning yet?](https://www.arewelearningyet.com/) being "Yes, and it's freaking fast!". Buy yourself a coffee -[^1]: - for more on supertraits see - [the advanced trait section of the rust book](https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#using-supertraits-to-require-one-traits-functionality-within-another-trait) +[^supertrait]: for more on supertraits see [the advanced trait section of the rust book](https://doc.rust-lang.org/book/ch19-03-advanced-traits.html#using-supertraits-to-require-one-traits-functionality-within-another-trait) -[^2]: - wiki link for - [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) +[^autodiff]: wiki link for [automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) -[^3]: - for more information on unit structs see - [the defining and instantiating structs section of the rust book](https://doc.rust-lang.org/book/ch05-01-defining-structs.html#unit-like-structs-without-any-fields) +[^absolute_units]: for more information on unit structs see [the defining and instantiating structs section of the rust book](https://doc.rust-lang.org/book/ch05-01-defining-structs.html#unit-like-structs-without-any-fields) From 6d9c7837e584423cbd1e4a35e82ee51e92eddacf Mon Sep 17 00:00:00 2001 From: Joshua Ferguson Date: Fri, 2 Feb 2024 18:04:50 -0600 Subject: [PATCH 32/32] implemented suggestions from code review --- .../src/getting-started/configuring-your-editor.md | 6 +++--- .../src/getting-started}/debug-options-vscode.png | Bin .../getting-started/setting-up-the-environment.md | 2 +- contributor-book/src/getting-started/testing.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename {assets => contributor-book/src/getting-started}/debug-options-vscode.png (100%) diff --git a/contributor-book/src/getting-started/configuring-your-editor.md b/contributor-book/src/getting-started/configuring-your-editor.md index 51d55de7e9..28eaff2e2d 100644 --- a/contributor-book/src/getting-started/configuring-your-editor.md +++ b/contributor-book/src/getting-started/configuring-your-editor.md @@ -19,9 +19,9 @@ To use the debugger, follow these steps: 2. Select the configuration from the "run and debug" side panel (it have a infested play button), then select the target from 3. Now you can enable breakpoint on code through IDE and then start debugging the library/binary you want, such as the following example: -
- -
+ +![debug-options](debug-options-vscode.png) + If you're creating a new library or binary, keep in mind to repeat the step 1. to always keep a fresh list of targets. diff --git a/assets/debug-options-vscode.png b/contributor-book/src/getting-started/debug-options-vscode.png similarity index 100% rename from assets/debug-options-vscode.png rename to contributor-book/src/getting-started/debug-options-vscode.png diff --git a/contributor-book/src/getting-started/setting-up-the-environment.md b/contributor-book/src/getting-started/setting-up-the-environment.md index 9f198377f6..2aaf2a8bf2 100644 --- a/contributor-book/src/getting-started/setting-up-the-environment.md +++ b/contributor-book/src/getting-started/setting-up-the-environment.md @@ -1,4 +1,4 @@ -# setting up the environment +# Setting up the environment There are a couple of tools that need to be installed, and commands to be familiar with, depending on what part of the project you plan on contributing to. This section should be up to date with diff --git a/contributor-book/src/getting-started/testing.md b/contributor-book/src/getting-started/testing.md index 2aa8a7bc26..2925386d30 100644 --- a/contributor-book/src/getting-started/testing.md +++ b/contributor-book/src/getting-started/testing.md @@ -1,4 +1,4 @@ -# testing +# Testing ## Test for TensorOps