autograd #

Node is a member of a computational graph that contains a reference to a gate, as well as the parents of the operation and the payload that resulted from the operation. Payload is a simple wrapper around a Variable. It is only abstracted out to be a bit more explicit that it is being passed around through an operation

fn abs_gate[T](a &Variable[T]) &AbsGate[T]

fn add_gate[T]() &AddGate[T]

fn clamp_gate[T](min_val T, max_val T, a &vtl.Tensor[T]) &ClampGate[T]

fn concat_gate[T](axis int, splits []int) &ConcatGate[T]

fn cos_gate[T](a &Variable[T]) &CosGate[T]

fn ctx[T]() &Context[T]

Contexts can only be initialized as empty, and a generic type must be provided

fn divide_gate[T](a &Variable[T], b &Variable[T]) &DivideGate[T]

fn exp_gate[T](a &Variable[T]) &ExpGate[T]

fn gate_backward[T](gate Gate[T], payload &Payload[T]) ![]&vtl.Tensor[T]

gate_backward is kept for backwards compatibility but now delegates directly through the Gate[T] interface instead of a manual match.

fn log_gate[T](a &Variable[T]) &LogGate[T]

fn matmul_gate[T](a &Variable[T], b &Variable[T]) &MatMulGate[T]

fn mean_gate[T](shape []int, axis int, num_elems int) &MeanGate[T]

fn multiply_gate[T](a &Variable[T], b &Variable[T]) &MultiplyGate[T]

fn node[T](gate Gate[T], parents []&Variable[T], payload &Payload[T], name string) &Node[T]

node

fn payload[T](variable &Variable[T]) &Payload[T]

fn pow_gate[T](a &Variable[T], b &Variable[T]) &PowGate[T]

fn register[T](name string, gate Gate[T], result &Variable[T], parents []&Variable[T]) !

fn reshape_gate[T](orig_shape []int) &ReshapeGate[T]

fn sin_gate[T](a &Variable[T]) &SinGate[T]

fn sqrt_gate[T](a &Variable[T]) &SqrtGate[T]

fn subtract_gate[T]() &SubtractGate[T]

fn sum_gate[T](shape []int, axis int) &SumGate[T]

fn tan_gate[T](a &Variable[T]) &TanGate[T]

fn tanh_gate[T](cache &vtl.Tensor[T]) &TanhGate[T]

fn transpose_gate[T](perm []int) &TransposeGate[T]

fn variable[T](context &Context[T], value &vtl.Tensor[T], data VariableData) &Variable[T]

variable

interface CacheParam {}

interface Gate[T] {
	backward(payload &Payload[T]) ![]&vtl.Tensor[T]
}

Gate is a generic interface for autograd operations. Any struct that implements backward for a given T satisfies Gate[T]. This allows both core math gates (AddGate, MatMulGate, …) and higher-level nn gates (LinearGate, ReLUGate, …) to participate in backpropagation without circular imports.

fn (g &AbsGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &AbsGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &AddGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &AddGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &ClampGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &ClampGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &ConcatGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &ConcatGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (mut ctx Context[T]) set_compute_backend(backend vtl.Backend)

set_compute_backend configures backend preference for runtime dispatch.

fn (mut ctx Context[T]) set_compute_strict(strict bool)

set_compute_strict toggles strict mode for runtime backend dispatch.

fn (ctx &Context[T]) len() int

fn (mut ctx Context[T]) push[T](node &Node[T])

fn (ctx &Context[T]) last[T]() !&Node[T]

fn (mut ctx Context[T]) pop[T]() !&Node[T]

fn (ctx &Context[T]) variable[T](value &vtl.Tensor[T], data ContextVariableData) &Variable[T]

fn (ctx &Context[T]) str() string

fn (g &CosGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &CosGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &DivideGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &DivideGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &ExpGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &ExpGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &LogGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &LogGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &MatMulGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &MatMulGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &MeanGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &MeanGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &MultiplyGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &MultiplyGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &PowGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &PowGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &ReshapeGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &ReshapeGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &SinGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &SinGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &SqrtGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &SqrtGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &SubtractGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &SubtractGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &SumGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &SumGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &TanGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &TanGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &TanhGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &TanhGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (g &TransposeGate[T]) backward[T](payload &Payload[T]) ![]&vtl.Tensor[T]

fn (g &TransposeGate[T]) cache[T](mut result Variable[T], args ...CacheParam) !

fn (v &Variable[T]) abs_op[T]() !&Variable[T]

abs_op computes the absolute value of the variable element-wise. Backward: grad * sign(x) (0 at x=0)

Named abs_op (not abs) to avoid collision with the built-in abs method on Tensor[T] which does not participate in the autograd graph.

x := ctx.variable(vtl.from_1d[f64]([-3.0, 0.0, 4.0]))
y := x.abs_op[f64]()!
// y.value = [3.0, 0.0, 4.0]

fn (v &Variable[T]) add[T](other &Variable[T]) !&Variable[T]

add Adds two variables together.

fn (mut v Variable[T]) backprop[T]() !

backprop Back propagates an operation along a computational graph. This operation will destroy the operational graph, populating the gradients for all variables that are predecessors of the Variable this is called on. Even if this is called on the first node in a graph, it will destroy all descendents of this variable stored by the Context

fn (v &Variable[T]) clamp[T](min_val T, max_val T) !&Variable[T]

clamp clips the variable element-wise to the range [min_val, max_val]. Backward: grad is passed through where min_val < x < max_val, zero otherwise.

x := ctx.variable(vtl.from_1d[f64]([-2.0, 0.5, 3.0]))
y := x.clamp[f64](-1.0, 1.0)!
// y.value = [-1.0, 0.5, 1.0]

fn (v &Variable[T]) cos[T]() !&Variable[T]

cos Cosine of a variable.

fn (v &Variable[T]) divide[T](other &Variable[T]) !&Variable[T]

divide Divides two variables.

fn (v &Variable[T]) exp[T]() !&Variable[T]

exp Exponentiates a variable.

fn (v &Variable[T]) is_grad_needed() bool

fn (v &Variable[T]) log[T]() !&Variable[T]

log computes the natural logarithm of the variable element-wise. Backward: grad * (1 / x)

Note: inputs must be positive; behaviour for x <= 0 is undefined.

x := ctx.variable(vtl.from_1d[f64]([1.0, math.e, math.exp(2.0)]))
y := x.log[f64]()!
// y.value ≈ [0.0, 1.0, 2.0]

fn (v &Variable[T]) matmul[T](other &Variable[T]) !&Variable[T]

matmul Multiplies two matrices.

fn (v &Variable[T]) multiply[T](other &Variable[T]) !&Variable[T]

multiply Multiplies two variables.

fn (v &Variable[T]) pow[T](other &Variable[T]) !&Variable[T]

pow raises a variable to a power.

fn (v &Variable[T]) reshape[T](new_shape []int) !&Variable[T]

reshape returns a new variable with the same data but a different shape. The total number of elements must be preserved. Backward: gradient is reshaped back to the original shape.

x := ctx.variable(vtl.from_array[f64]([1.0, 2.0, 3.0, 4.0], [2, 2]))
y := x.reshape[f64]([4])!
// y.value.shape = [4]

fn (v &Variable[T]) sin[T]() !&Variable[T]

sin Sine of a variable.

fn (v &Variable[T]) slice[T](idx ...[]int) !&Variable[T]

fn (v &Variable[T]) slice_hilo[T](idx1 []int, idx2 []int) !&Variable[T]

fn (v &Variable[T]) sqrt_op[T]() !&Variable[T]

sqrt_op computes the element-wise square root of the variable. Backward: grad * (1 / (2 * sqrt(x)))

Note: inputs must be non-negative.

Named sqrt_op (not sqrt) to avoid collision with Tensor.sqrt which does not participate in the autograd graph.

x := ctx.variable(vtl.from_1d[f64]([1.0, 4.0, 9.0]))
y := x.sqrt_op[f64]()!
// y.value = [1.0, 2.0, 3.0]

fn (v &Variable[T]) str() string

fn (v &Variable[T]) subtract[T](other &Variable[T]) !&Variable[T]

subtract Subtracts two variables.

fn (v &Variable[T]) tan[T]() !&Variable[T]

tan Tan of a variable.

fn (v &Variable[T]) tanh_op[T]() !&Variable[T]

tanh_op computes the element-wise hyperbolic tangent of the variable. Backward: grad * (1 - tanh²(x))

Named tanh_op (not tanh) to avoid collision with Tensor.tanh which does not participate in the autograd graph.

x := ctx.variable(vtl.from_1d[f64]([0.0, 1.0, -1.0]))
y := x.tanh_op[f64]()!
// y.value ≈ [0.0, 0.762, -0.762]

fn (v &Variable[T]) transpose_op[T](perm []int) !&Variable[T]

transpose_op permutes the axes of the variable according to perm. perm must be a permutation of [0, 1, ..., ndim-1]. Backward: gradient is transposed with the inverse permutation.

Named transpose_op (not transpose) to avoid collision with Tensor.transpose.

x := ctx.variable(vtl.from_array[f64]([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], [2, 3]))
y := x.transpose_op[f64]([1, 0])!
// y.value.shape = [3, 2]

struct AbsGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
}

AbsGate implements abs(x) element-wise. backward: grad * sign(x)

struct AddGate[T] {}

struct ClampGate[T] {
pub:
	min_val T
	max_val T
	a       &vtl.Tensor[T] = unsafe { nil }
}

ClampGate implements clamp(x, min, max) element-wise. backward: grad where x is within [min, max], 0 otherwise

struct ConcatGate[T] {
pub:
	axis   int
	splits []int // size of each input along the concat axis
}

ConcatGate concatenates multiple tensors along an axis. backward: split gradient back into original inputs

struct Context[T] {
pub mut:
	// A list of all variables present in an operation.
	// This list can contain duplicates
	nodes []&Node[T]
	// If no_grad is set to true, operations will not
	// be cached, and backpropagation will not be possible
	no_grad bool
	// Backend preference used by NN/LA runtime dispatch.
	compute_backend vtl.Backend = .auto
	// If true, fail when preferred backend is unavailable.
	compute_strict bool
}

Context keeps track of the computational graph for a number of operations. Variables that interact with each other must belong to the same context, or state will be lost while tracking operations done.

struct ContextVariableData {
pub:
	requires_grad bool = true
}

struct CosGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
}

struct DivideGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
	b &Variable[T] = unsafe { nil }
}

struct ExpGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
}

struct LogGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
}

LogGate implements log(x) element-wise. backward: grad * (1/x)

struct MatMulGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
	b &Variable[T] = unsafe { nil }
}

struct MeanGate[T] {
pub:
	shape     []int
	axis      int
	num_elems int
}

MeanGate implements mean reduction. backward: grad broadcast to input shape / num_elements

struct MultiplyGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
	b &Variable[T] = unsafe { nil }
}

struct Node[T] {
pub:
	// A Gate[T] containing a backward function for this node.
	// Using the generic interface allows any gate (core math or nn)
	// to participate in backpropagation without circular imports.
	gate Gate[T]
pub mut:
	// The variables that created this node
	parents []&Variable[T]
	// Wrapper around a Tensor, contains operation data
	payload &Payload[T] = unsafe { nil }
	// Debug use only, contains a name for a node
	name string
}

struct Payload[T] {
pub:
	// Contents of the paylod
	variable &Variable[T] = unsafe { nil }
}

struct PowGate[T] {
	a &Variable[T] = unsafe { nil }
	b &Variable[T] = unsafe { nil }
}

struct ReshapeGate[T] {
pub:
	orig_shape []int
}

ReshapeGate stores the original shape for backward pass. backward: grad reshaped back to original shape

struct SinGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
}

struct SqrtGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
}

SqrtGate implements sqrt(x) element-wise. backward: grad * (1 / (2 * sqrt(x)))

struct SubtractGate[T] {}

struct SumGate[T] {
pub:
	shape []int
	axis  int
}

SumGate implements sum reduction. backward: grad broadcast back to input shape

struct TanGate[T] {
pub:
	a &Variable[T] = unsafe { nil }
}

struct TanhGate[T] {
pub:
	cache &vtl.Tensor[T] = unsafe { nil }
}

TanhGate implements tanh(x) element-wise. backward: grad * (1 - tanh(x)^2) = grad * (1 - cached^2)

struct TransposeGate[T] {
pub:
	perm  []int
	iperm []int
}

TransposeGate stores the permutation for backward. backward: grad transposed back with inverse permutation

struct Variable[T] {
pub mut:
	// The value of the Variable.  This should not be edited outside
	// of Variable operations, as other edits will not be tracked
	// and will lead to incorrect results
	value &vtl.Tensor[T] = unsafe { nil }
	// The graph the variable is associated with.  This is a reference,
	// as a variable does not own its context
	context &Context[T] = unsafe { nil }
	// The gradient of the Variable.  This is set as a reference to
	// the value of a Variable unless `backprop` has been called, in
	// which case all related Variables will have their gradient
	// updated correctly
	grad &vtl.Tensor[T] = unsafe { nil }
	// If set to true, this variable will track its operations,
	// otherwise it will act similar to a vtl.Tensor, only calculating
	// forward operations
	requires_grad bool
}

Variable is an abstraction of a vtl.Tensor that tracks the operations done to the vtl.Tensor. It also keeps track of the gradient of the operation if a Variable needs to backpropagate. This is the fundamental object used in automatic differentiation, as well as the neural network aspects of VTL

struct VariableData {
	requires_grad bool = true
}