From 8c6a43e8601da241344660caa3cfc890c12e922e Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Sat, 17 Jan 2026 09:03:23 +0100 Subject: [PATCH 01/21] add `arrayify` for adjoint tensor --- ext/TensorKitMooncakeExt/tangent.jl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ext/TensorKitMooncakeExt/tangent.jl b/ext/TensorKitMooncakeExt/tangent.jl index 761e626f0..9fa6e401a 100644 --- a/ext/TensorKitMooncakeExt/tangent.jl +++ b/ext/TensorKitMooncakeExt/tangent.jl @@ -5,3 +5,11 @@ function Mooncake.arrayify(A_dA::CoDual{<:TensorMap}) dA = typeof(A)(data, A.space) return A, dA end + +function Mooncake.arrayify(Aᴴ_ΔAᴴ::CoDual{<:TensorKit.AdjointTensorMap}) + Aᴴ = Mooncake.primal(Aᴴ_ΔAᴴ) + ΔAᴴ = Mooncake.tangent(Aᴴ_ΔAᴴ) + A_ΔA = CoDual(Aᴴ', ΔAᴴ.data.parent) + A, ΔA = arrayify(A_ΔA) + return A', ΔA' +end From 677171ac3363783e01eea704ac7f1a659fb7e8f0 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Sat, 17 Jan 2026 09:03:29 +0100 Subject: [PATCH 02/21] add vectorinterface rules --- .../TensorKitMooncakeExt.jl | 6 +- ext/TensorKitMooncakeExt/vectorinterface.jl | 93 +++++++++++++++++++ test/autodiff/mooncake.jl | 25 +++++ 3 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 ext/TensorKitMooncakeExt/vectorinterface.jl diff --git a/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl b/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl index b35c73f4c..2cc64f49b 100644 --- a/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl +++ b/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl @@ -1,17 +1,17 @@ module TensorKitMooncakeExt using Mooncake -using Mooncake: @zero_derivative, DefaultCtx, ReverseMode, NoRData, CoDual, arrayify, primal +using Mooncake: @zero_derivative, DefaultCtx, ReverseMode, NoFData, NoRData, CoDual, arrayify, primal using TensorKit +using VectorInterface using TensorOperations: TensorOperations, IndexTuple, Index2Tuple, linearize import TensorOperations as TO -using VectorInterface: One, Zero using TupleTools - include("utility.jl") include("tangent.jl") include("linalg.jl") +include("vectorinterface.jl") include("tensoroperations.jl") end diff --git a/ext/TensorKitMooncakeExt/vectorinterface.jl b/ext/TensorKitMooncakeExt/vectorinterface.jl new file mode 100644 index 000000000..2c1bfe984 --- /dev/null +++ b/ext/TensorKitMooncakeExt/vectorinterface.jl @@ -0,0 +1,93 @@ +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(scale!), AbstractTensorMap, Number} + +function Mooncake.rrule!!(::CoDual{typeof(scale!)}, C_ΔC::CoDual{<:AbstractTensorMap}, α_Δα::CoDual{<:Number}) + # prepare arguments + C, ΔC = arrayify(C_ΔC) + α = primal(α_Δα) + + # primal call + C_cache = copy(C) + scale!(C, α) + + function scale_pullback(::NoRData) + copy!(C, C_cache) + scale!(ΔC, conj(α)) + TΔα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) + Δαr = TΔα === NoRData ? NoRData() : inner(C, ΔC) + return NoRData(), NoRData(), Δαr + end + + return C_ΔC, scale_pullback +end + +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(scale!), AbstractTensorMap, AbstractTensorMap, Number} + +function Mooncake.rrule!!(::CoDual{typeof(scale!)}, C_ΔC::CoDual{<:AbstractTensorMap}, A_ΔA::CoDual{<:AbstractTensorMap}, α_Δα::CoDual{<:Number}) + # prepare arguments + C, ΔC = arrayify(C_ΔC) + A, ΔA = arrayify(A_ΔA) + α = primal(α_Δα) + + # primal call + C_cache = copy(C) + scale!(C, A, α) + + function scale_pullback(::NoRData) + copy!(C, C_cache) + zerovector!(ΔC) + scale!(ΔA, conj(α)) + TΔα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) + Δαr = TΔα === NoRData ? NoRData() : inner(C, ΔC) + return NoRData(), NoRData(), NoRData(), Δαr + end + + return C_ΔC, scale_pullback +end + +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(add!), AbstractTensorMap, AbstractTensorMap, Number, Number} + +function Mooncake.rrule!!(::CoDual{typeof(add!)}, C_ΔC::CoDual{<:AbstractTensorMap}, A_ΔA::CoDual{<:AbstractTensorMap}, α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}) + # prepare arguments + C, ΔC = arrayify(C_ΔC) + A, ΔA = arrayify(A_ΔA) + α = primal(α_Δα) + β = primal(β_Δβ) + + # primal call + C_cache = copy(C) + add!(C, A, α, β) + + function add_pullback(::NoRData) + copy!(C, C_cache) + scale!(ΔC, conj(β)) + scale!(ΔA, conj(α)) + + TΔα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) + Δαr = TΔα === NoRData ? NoRData() : inner(A, ΔC) + TΔβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) + Δβr = TΔβ === NoRData ? NoRData() : inner(C, ΔC) + + return NoRData(), NoRData(), NoRData(), Δαr, Δβr + end + + return C_ΔC, add_pullback +end + +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(inner), AbstractTensorMap, AbstractTensorMap} + +function Mooncake.rrule!!(::CoDual{typeof(inner)}, A_ΔA::CoDual{<:AbstractTensorMap}, B_ΔB::CoDual{<:AbstractTensorMap}) + # prepare arguments + A, ΔA = arrayify(A_ΔA) + B, ΔB = arrayify(B_ΔB) + + # primal call + s = inner(A, B) + + function inner_pullback(Δs) + scale!(ΔA, B, conj(Δs)) + scale!(ΔB, A, Δs) + return NoRData(), NoRData(), NoRData() + end + + return CoDual(s, NoFData()), inner_pullback +end diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index 1cd74fa27..37eb932b5 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -68,6 +68,31 @@ for V in spacelist println("Mooncake with symmetry: $Istr") println("---------------------------------------") eltypes = (Float64,) # no complex support yet + + @timedtestset "VectorInterface with scalartype $T" for T in eltypes + atol = precision(T) + rtol = precision(T) + + C = randn(T, V[1] ⊗ V[2] ← V[3] ⊗ V[4] ⊗ V[5]) + A = randn(T, V[1] ⊗ V[2] ← V[3] ⊗ V[4] ⊗ V[5]) + α = randn(T) + β = randn(T) + + Mooncake.TestUtils.test_rule(rng, scale!, C, α; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, scale!, C', α; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, scale!, C, A, α; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, scale!, C', A', α; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, scale!, copy(C'), A', α; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, scale!, C', copy(A'), α; atol, rtol, mode) + + Mooncake.TestUtils.test_rule(rng, add!, C, A; atol, rtol, mode, is_primitive = false) + Mooncake.TestUtils.test_rule(rng, add!, C, A, α; atol, rtol, mode, is_primitive = false) + Mooncake.TestUtils.test_rule(rng, add!, C, A, α, β; atol, rtol, mode) + + Mooncake.TestUtils.test_rule(rng, inner, C, A; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, inner, C', A'; atol, rtol, mode) + end + symmetricbraiding && @timedtestset "TensorOperations with scalartype $T" for T in eltypes atol = precision(T) rtol = precision(T) From e25c22951bb2784d10fe70d4d7a97d08d4112ca5 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Sat, 17 Jan 2026 12:37:19 +0100 Subject: [PATCH 03/21] add tensoroperations rules --- ext/TensorKitMooncakeExt/tensoroperations.jl | 178 +++++++++++++++++++ test/autodiff/mooncake.jl | 44 +++++ 2 files changed, 222 insertions(+) diff --git a/ext/TensorKitMooncakeExt/tensoroperations.jl b/ext/TensorKitMooncakeExt/tensoroperations.jl index d663a3281..7b9a674f8 100644 --- a/ext/TensorKitMooncakeExt/tensoroperations.jl +++ b/ext/TensorKitMooncakeExt/tensoroperations.jl @@ -1,3 +1,91 @@ +# tensoradd! +# ---------- +Mooncake.@is_primitive( + DefaultCtx, + ReverseMode, + Tuple{ + typeof(TO.tensoradd!), + AbstractTensorMap, + AbstractTensorMap, Index2Tuple, Bool, + Number, Number, Vararg{Any}, + } +) + +function Mooncake.rrule!!( + ::CoDual{typeof(TO.tensoradd!)}, + C_ΔC::CoDual{<:AbstractTensorMap}, + A_ΔA::CoDual{<:AbstractTensorMap}, pA_ΔpA::CoDual{<:Index2Tuple}, conjA_ΔconjA::CoDual{Bool}, + α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, + ba_Δba::CoDual... + ) + # prepare arguments + C, ΔC = arrayify(C_ΔC) + A, ΔA = arrayify(A_ΔA) + pA = primal(pA_ΔpA) + conjA = primal(conjA_ΔconjA) + α, β = primal.((α_Δα, β_Δβ)) + ba = primal.(ba_Δba) + + # primal call + C_cache = copy(C) + TO.tensoradd!(C, A, pA, conjA, α, β, ba...) + + function tensoradd_pullback(::NoRData) + copy!(C, C_cache) + + ΔCr = tensoradd_pullback_ΔC!(ΔC, β) + ΔAr = tensoradd_pullback_ΔA!(ΔA, ΔC, A, pA, conjA, α, ba...) + Δαr = tensoradd_pullback_Δα(ΔC, A, pA, conjA, α, ba...) + Δβr = tensoradd_pullback_Δβ(ΔC, C, β) + + return NoRData(), + ΔCr, + ΔAr, NoRData(), NoRData(), + Δαr, Δβr, + map(Returns(NoRData()), ba)... + end + + return C_ΔC, tensoradd_pullback +end + +tensoradd_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) + +function tensoradd_pullback_ΔA!( + ΔA, ΔC, A, pA, conjA, α, ba... + ) + ipA = invperm(linearize(pA)) + pΔA = _repartition(ipA, A) + TO.tensoradd!(ΔA, ΔC, pΔA, conjA, conjA ? α : conj(α), Zero(), ba...) + return NoRData() +end + +function tensoradd_pullback_Δα( + ΔC, A, pA, conjA, α, ba... + ) + Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) + Tdα === NoRData && return NoRData() + + tΔC = twist(ΔC, filter(x -> isdual(space(ΔC, x)), allind(ΔC)); copy = false) + Δα = TO.tensorscalar( + TO.tensorcontract( + A, ((), linearize(pA)), !conjA, + tΔC, (trivtuple(TO.numind(pA)), ()), false, + ((), ()), One(), ba... + ) + ) + return Mooncake._rdata(Δα) +end + +function tensoradd_pullback_Δβ(ΔC, C, β) + Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) + Tdβ === NoRData && return NoRData() + + Δβ = inner(C, ΔC) + return Mooncake._rdata(Δβ) +end + +# tensorcontract! +# --------------- Mooncake.@is_primitive( DefaultCtx, ReverseMode, @@ -135,3 +223,93 @@ function tensorcontract_pullback_Δβ(ΔC, C, β) Δβ = inner(C, ΔC) return Mooncake._rdata(Δβ) end + +# tensortrace! +# ------------ +Mooncake.@is_primitive( + DefaultCtx, + ReverseMode, + Tuple{ + typeof(TO.tensortrace!), + AbstractTensorMap, + AbstractTensorMap, Index2Tuple, Index2Tuple, Bool, + Number, Number, + Vararg{Any}, + } +) + +function Mooncake.rrule!!( + ::CoDual{typeof(TO.tensortrace!)}, + C_ΔC::CoDual{<:AbstractTensorMap}, + A_ΔA::CoDual{<:AbstractTensorMap}, p_Δp::CoDual{<:Index2Tuple}, q_Δq::CoDual{<:Index2Tuple}, conjA_ΔconjA::CoDual{Bool}, + α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, + ba_Δba::CoDual... + ) + # prepare arguments + C, ΔC = arrayify(C_ΔC) + A, ΔA = arrayify(A_ΔA) + p = primal(p_Δp) + q = primal(q_Δq) + conjA = primal(conjA_ΔconjA) + α, β = primal.((α_Δα, β_Δβ)) + ba = primal.(ba_Δba) + + # primal call + C_cache = copy(C) + TO.tensortrace!(C, A, p, q, conjA, α, β, ba...) + + function tensortrace_pullback(::NoRData) + copy!(C, C_cache) + + ΔCr = tensortrace_pullback_ΔC!(ΔC, β) + ΔAr = tensortrace_pullback_ΔA!(ΔA, ΔC, A, p, q, conjA, α, ba...) + Δαr = tensortrace_pullback_Δα(ΔC, A, p, q, conjA, α, ba...) + Δβr = tensortrace_pullback_Δβ(ΔC, C, β) + + return NoRData(), + ΔCr, + ΔAr, NoRData(), NoRData(), NoRData(), + Δαr, Δβr, + map(Returns(NoRData()), ba)... + end + + return C_ΔC, tensortrace_pullback +end + +tensortrace_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) + +function tensortrace_pullback_ΔA!( + ΔA, ΔC, A, p, q, conjA, α, ba... + ) + ip = invperm((linearize(p)..., q[1]..., q[2]...)) + pdA = _repartition(ip, A) + E = one!(TO.tensoralloc_add(scalartype(A), A, q, conjA)) + twist!(E, filter(x -> !isdual(space(E, x)), codomainind(E))) + pE = ((), trivtuple(TO.numind(q))) + pΔC = (trivtuple(TO.numind(p)), ()) + TO.tensorproduct!( + ΔA, ΔC, pΔC, conjA, E, pE, conjA, pdA, conjA ? α : conj(α), Zero(), ba... + ) + return NoRData() +end + +function tensortrace_pullback_Δα( + ΔC, A, p, q, conjA, α, ba... + ) + Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) + Tdα === NoRData && return NoRData() + + # TODO: this result might be easier to compute as: + # C′ = βC + α * trace(A) ⟹ At = (C′ - βC) / α + At = TO.tensortrace(A, p, q, conjA) + Δα = inner(At, ΔC) + return Mooncake._rdata(Δα) +end + +function tensortrace_pullback_Δβ(ΔC, C, β) + Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) + Tdβ === NoRData && return NoRData() + + Δβ = inner(C, ΔC) + return Mooncake._rdata(Δβ) +end diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index 37eb932b5..38fa23c15 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -97,6 +97,25 @@ for V in spacelist atol = precision(T) rtol = precision(T) + @timedtestset "tensoradd!" begin + A = randn(T, V[1] ⊗ V[2] ← V[4] ⊗ V[5]) + α = randn(T) + β = randn(T) + + # repeat a couple times to get some distribution of arrows + for _ in 1:5 + p = randindextuple(numind(A)) + + C1 = randn!(TensorOperations.tensoralloc_add(T, A, p, false, Val(false))) + Mooncake.TestUtils.test_rule(rng, tensoradd!, C1, A, p, false, α, β; atol, rtol, mode) + + C2 = randn!(TensorOperations.tensoralloc_add(T, A, p, true, Val(false))) + Mooncake.TestUtils.test_rule(rng, tensoradd!, C2, A, p, true, α, β; atol, rtol, mode) + + A = rand(Bool) ? C1 : C2 + end + end + @timedtestset "tensorcontract!" begin for _ in 1:5 d = 0 @@ -138,5 +157,30 @@ for V in spacelist end end end + + @timedtestset "tensortrace!" begin + for _ in 1:5 + k1 = rand(0:2) + k2 = rand(1:2) + V1 = map(v -> rand(Bool) ? v' : v, rand(V, k1)) + V2 = map(v -> rand(Bool) ? v' : v, rand(V, k2)) + + (_p, _q) = randindextuple(k1 + 2 * k2, k1) + p = _repartition(_p, rand(0:k1)) + q = _repartition(_q, k2) + ip = _repartition(invperm(linearize((_p, _q))), rand(0:(k1 + 2 * k2))) + A = randn(T, permute(prod(V1) ⊗ prod(V2) ← prod(V2), ip)) + + α = randn(T) + β = randn(T) + for conjA in (false, true) + C = randn!(TensorOperations.tensoralloc_add(T, A, p, conjA, Val(false))) + Mooncake.TestUtils.test_rule( + rng, tensortrace!, C, A, p, q, conjA, α, β; + atol, rtol, mode, is_primitive = false + ) + end + end + end end end From e6338c42598bc8c1a585173c8a228d1aa85fdab8 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Sun, 18 Jan 2026 08:49:45 -0500 Subject: [PATCH 04/21] add indexmanipulations rules --- .../TensorKitMooncakeExt.jl | 2 + .../indexmanipulations.jl | 153 ++++++++++++++++ ext/TensorKitMooncakeExt/tensoroperations.jl | 166 +++++++++--------- test/autodiff/mooncake.jl | 61 ++++++- 4 files changed, 297 insertions(+), 85 deletions(-) create mode 100644 ext/TensorKitMooncakeExt/indexmanipulations.jl diff --git a/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl b/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl index 2cc64f49b..15e0c4c9f 100644 --- a/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl +++ b/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl @@ -3,6 +3,7 @@ module TensorKitMooncakeExt using Mooncake using Mooncake: @zero_derivative, DefaultCtx, ReverseMode, NoFData, NoRData, CoDual, arrayify, primal using TensorKit +import TensorKit as TK using VectorInterface using TensorOperations: TensorOperations, IndexTuple, Index2Tuple, linearize import TensorOperations as TO @@ -11,6 +12,7 @@ using TupleTools include("utility.jl") include("tangent.jl") include("linalg.jl") +include("indexmanipulations.jl") include("vectorinterface.jl") include("tensoroperations.jl") diff --git a/ext/TensorKitMooncakeExt/indexmanipulations.jl b/ext/TensorKitMooncakeExt/indexmanipulations.jl new file mode 100644 index 000000000..a0b73dde2 --- /dev/null +++ b/ext/TensorKitMooncakeExt/indexmanipulations.jl @@ -0,0 +1,153 @@ +for transform in (:permute, :transpose) + add_transform! = Symbol(:add_, transform, :!) + add_transform_pullback = Symbol(add_transform!, :_pullback) + @eval Mooncake.@is_primitive( + DefaultCtx, + ReverseMode, + Tuple{ + typeof(TK.$add_transform!), + AbstractTensorMap, + AbstractTensorMap, Index2Tuple, + Number, Number, Vararg{Any}, + } + ) + + @eval function Mooncake.rrule!!( + ::CoDual{typeof(TK.$add_transform!)}, + C_ΔC::CoDual{<:AbstractTensorMap}, + A_ΔA::CoDual{<:AbstractTensorMap}, p_Δp::CoDual{<:Index2Tuple}, + α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, + ba_Δba::CoDual... + ) + # prepare arguments + C, ΔC = arrayify(C_ΔC) + A, ΔA = arrayify(A_ΔA) + p = primal(p_Δp) + α, β = primal.((α_Δα, β_Δβ)) + ba = primal.(ba_Δba) + + C_cache = copy(C) + + # if we need to compute Δa, it is faster to allocate an intermediate permuted A + # and store that instead of repeating the permutation in the pullback each time. + # effectively, we replace `add_permute` by `add ∘ permute`. + Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) + Ap = if Tdα === NoRData + TK.$add_transform!(C, A, p, α, β, ba...) + nothing + else + Ap = $transform(A, p) + add!(C, Ap, α, β) + Ap + end + + function $add_transform_pullback(::NoRData) + copy!(C, C_cache) + + scale!(ΔC, conj(β)) + ΔCr = NoRData() + + # ΔA + ip = invperm(linearize(p)) + pΔA = _repartition(ip, A) + TK.$add_transform!(ΔA, ΔC, pΔA, conj(α), One(), ba...) + ΔAr = NoRData() + + # Δα + Δαr = if isnothing(Ap) + NoRData() + else + Mooncake._rdata(inner(Ap, ΔC)) + end + + # Δβ + Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) + Δβr = if Tdβ === NoRData + NoRData() + else + Mooncake._rdata(inner(C, ΔC)) + end + + + return NoRData(), ΔCr, ΔAr, NoRData(), Δαr, Δβr, map(Returns(NoRData()), ba)... + end + + return C_ΔC, $add_transform_pullback + end +end + +Mooncake.@is_primitive( + DefaultCtx, + ReverseMode, + Tuple{ + typeof(TK.add_braid!), + AbstractTensorMap, + AbstractTensorMap, Index2Tuple, IndexTuple, + Number, Number, Vararg{Any}, + } +) + +function Mooncake.rrule!!( + ::CoDual{typeof(TK.add_braid!)}, + C_ΔC::CoDual{<:AbstractTensorMap}, + A_ΔA::CoDual{<:AbstractTensorMap}, p_Δp::CoDual{<:Index2Tuple}, levels_Δlevels::CoDual{<:IndexTuple}, + α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, + ba_Δba::CoDual... + ) + # prepare arguments + C, ΔC = arrayify(C_ΔC) + A, ΔA = arrayify(A_ΔA) + p = primal(p_Δp) + levels = primal(levels_Δlevels) + α, β = primal.((α_Δα, β_Δβ)) + ba = primal.(ba_Δba) + + C_cache = copy(C) + + # if we need to compute Δa, it is faster to allocate an intermediate braided A + # and store that instead of repeating the permutation in the pullback each time. + # effectively, we replace `add_permute` by `add ∘ permute`. + Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) + Ap = if Tdα === NoRData + TK.add_braid!(C, A, p, levels, α, β, ba...) + nothing + else + Ap = braid(A, p, levels) + add!(C, Ap, α, β) + Ap + end + + function add_braid!_pullback(::NoRData) + copy!(C, C_cache) + + scale!(ΔC, conj(β)) + ΔCr = NoRData() + + # ΔA + ip = invperm(linearize(p)) + pΔA = _repartition(ip, A) + ilevels = TupleTools.permute(levels, linearize(p)) + TK.add_braid!(ΔA, ΔC, pΔA, ilevels, conj(α), One(), ba...) + ΔAr = NoRData() + + # Δα + Δαr = if isnothing(Ap) + NoRData() + else + Mooncake._rdata(inner(Ap, ΔC)) + end + + # Δβ + Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) + Δβr = if Tdβ === NoRData + NoRData() + else + Mooncake._rdata(inner(C, ΔC)) + end + + + return NoRData(), ΔCr, ΔAr, NoRData(), NoRData(), Δαr, Δβr, map(Returns(NoRData()), ba)... + end + + return C_ΔC, add_braid!_pullback +end diff --git a/ext/TensorKitMooncakeExt/tensoroperations.jl b/ext/TensorKitMooncakeExt/tensoroperations.jl index 7b9a674f8..915a10356 100644 --- a/ext/TensorKitMooncakeExt/tensoroperations.jl +++ b/ext/TensorKitMooncakeExt/tensoroperations.jl @@ -1,88 +1,88 @@ # tensoradd! # ---------- -Mooncake.@is_primitive( - DefaultCtx, - ReverseMode, - Tuple{ - typeof(TO.tensoradd!), - AbstractTensorMap, - AbstractTensorMap, Index2Tuple, Bool, - Number, Number, Vararg{Any}, - } -) - -function Mooncake.rrule!!( - ::CoDual{typeof(TO.tensoradd!)}, - C_ΔC::CoDual{<:AbstractTensorMap}, - A_ΔA::CoDual{<:AbstractTensorMap}, pA_ΔpA::CoDual{<:Index2Tuple}, conjA_ΔconjA::CoDual{Bool}, - α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, - ba_Δba::CoDual... - ) - # prepare arguments - C, ΔC = arrayify(C_ΔC) - A, ΔA = arrayify(A_ΔA) - pA = primal(pA_ΔpA) - conjA = primal(conjA_ΔconjA) - α, β = primal.((α_Δα, β_Δβ)) - ba = primal.(ba_Δba) - - # primal call - C_cache = copy(C) - TO.tensoradd!(C, A, pA, conjA, α, β, ba...) - - function tensoradd_pullback(::NoRData) - copy!(C, C_cache) - - ΔCr = tensoradd_pullback_ΔC!(ΔC, β) - ΔAr = tensoradd_pullback_ΔA!(ΔA, ΔC, A, pA, conjA, α, ba...) - Δαr = tensoradd_pullback_Δα(ΔC, A, pA, conjA, α, ba...) - Δβr = tensoradd_pullback_Δβ(ΔC, C, β) - - return NoRData(), - ΔCr, - ΔAr, NoRData(), NoRData(), - Δαr, Δβr, - map(Returns(NoRData()), ba)... - end - - return C_ΔC, tensoradd_pullback -end - -tensoradd_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) - -function tensoradd_pullback_ΔA!( - ΔA, ΔC, A, pA, conjA, α, ba... - ) - ipA = invperm(linearize(pA)) - pΔA = _repartition(ipA, A) - TO.tensoradd!(ΔA, ΔC, pΔA, conjA, conjA ? α : conj(α), Zero(), ba...) - return NoRData() -end - -function tensoradd_pullback_Δα( - ΔC, A, pA, conjA, α, ba... - ) - Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) - Tdα === NoRData && return NoRData() - - tΔC = twist(ΔC, filter(x -> isdual(space(ΔC, x)), allind(ΔC)); copy = false) - Δα = TO.tensorscalar( - TO.tensorcontract( - A, ((), linearize(pA)), !conjA, - tΔC, (trivtuple(TO.numind(pA)), ()), false, - ((), ()), One(), ba... - ) - ) - return Mooncake._rdata(Δα) -end - -function tensoradd_pullback_Δβ(ΔC, C, β) - Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) - Tdβ === NoRData && return NoRData() - - Δβ = inner(C, ΔC) - return Mooncake._rdata(Δβ) -end +# Mooncake.@is_primitive( +# DefaultCtx, +# ReverseMode, +# Tuple{ +# typeof(TO.tensoradd!), +# AbstractTensorMap, +# AbstractTensorMap, Index2Tuple, Bool, +# Number, Number, Vararg{Any}, +# } +# ) +# +# function Mooncake.rrule!!( +# ::CoDual{typeof(TO.tensoradd!)}, +# C_ΔC::CoDual{<:AbstractTensorMap}, +# A_ΔA::CoDual{<:AbstractTensorMap}, pA_ΔpA::CoDual{<:Index2Tuple}, conjA_ΔconjA::CoDual{Bool}, +# α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, +# ba_Δba::CoDual... +# ) +# # prepare arguments +# C, ΔC = arrayify(C_ΔC) +# A, ΔA = arrayify(A_ΔA) +# pA = primal(pA_ΔpA) +# conjA = primal(conjA_ΔconjA) +# α, β = primal.((α_Δα, β_Δβ)) +# ba = primal.(ba_Δba) +# +# # primal call +# C_cache = copy(C) +# TO.tensoradd!(C, A, pA, conjA, α, β, ba...) +# +# function tensoradd_pullback(::NoRData) +# copy!(C, C_cache) +# +# ΔCr = tensoradd_pullback_ΔC!(ΔC, β) +# ΔAr = tensoradd_pullback_ΔA!(ΔA, ΔC, A, pA, conjA, α, ba...) +# Δαr = tensoradd_pullback_Δα(ΔC, A, pA, conjA, α, ba...) +# Δβr = tensoradd_pullback_Δβ(ΔC, C, β) +# +# return NoRData(), +# ΔCr, +# ΔAr, NoRData(), NoRData(), +# Δαr, Δβr, +# map(Returns(NoRData()), ba)... +# end +# +# return C_ΔC, tensoradd_pullback +# end +# +# tensoradd_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) +# +# function tensoradd_pullback_ΔA!( +# ΔA, ΔC, A, pA, conjA, α, ba... +# ) +# ipA = invperm(linearize(pA)) +# pΔA = _repartition(ipA, A) +# TO.tensoradd!(ΔA, ΔC, pΔA, conjA, conjA ? α : conj(α), Zero(), ba...) +# return NoRData() +# end +# +# function tensoradd_pullback_Δα( +# ΔC, A, pA, conjA, α, ba... +# ) +# Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) +# Tdα === NoRData && return NoRData() +# +# tΔC = twist(ΔC, filter(x -> isdual(space(ΔC, x)), allind(ΔC)); copy = false) +# Δα = TO.tensorscalar( +# TO.tensorcontract( +# A, ((), linearize(pA)), !conjA, +# tΔC, (trivtuple(TO.numind(pA)), ()), false, +# ((), ()), One(), ba... +# ) +# ) +# return Mooncake._rdata(Δα) +# end +# +# function tensoradd_pullback_Δβ(ΔC, C, β) +# Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) +# Tdβ === NoRData && return NoRData() +# +# Δβ = inner(C, ΔC) +# return Mooncake._rdata(Δβ) +# end # tensorcontract! # --------------- diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index 38fa23c15..2ca21654e 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -3,6 +3,7 @@ using TensorKit using TensorOperations using Mooncake using Random +using TupleTools mode = Mooncake.ReverseMode rng = Random.default_rng() @@ -13,6 +14,14 @@ function randindextuple(N::Int, k::Int = rand(0:N)) _p = randperm(N) return (tuple(_p[1:k]...), tuple(_p[(k + 1):end]...)) end +function randcircshift(N₁::Int, N₂::Int, k::Int = rand(0:(N₁ + N₂))) + N = N₁ + N₂ + @assert 0 ≤ k ≤ N + p = TupleTools.vcat(ntuple(identity, N₁), reverse(ntuple(identity, N₂) .+ N₁)) + n = rand(0:N) + _p = TupleTools.circshift(p, n) + return (tuple(_p[1:k]...), reverse(tuple(_p[(k + 1):end]...))) +end const _repartition = @static if isdefined(Base, :get_extension) Base.get_extension(TensorKit, :TensorKitMooncakeExt)._repartition @@ -93,6 +102,54 @@ for V in spacelist Mooncake.TestUtils.test_rule(rng, inner, C', A'; atol, rtol, mode) end + @timedtestset "Index manipulations with scalartype $T" for T in eltypes + atol = precision(T) + rtol = precision(T) + + symmetricbraiding && @timedtestset "add_permute!" begin + A = randn(T, V[1] ⊗ V[2] ← V[4] ⊗ V[5]) + α = randn(T) + β = randn(T) + + # repeat a couple times to get some distribution of arrows + for _ in 1:5 + p = randindextuple(numind(A)) + C = randn!(permute(A, p)) + Mooncake.TestUtils.test_rule(rng, TensorKit.add_permute!, C, A, p, α, β; atol, rtol, mode) + A = C + end + end + + @timedtestset "add_transpose!" begin + A = randn(T, V[1] ⊗ V[2] ← V[4] ⊗ V[5]) + α = randn(T) + β = randn(T) + + # repeat a couple times to get some distribution of arrows + for _ in 1:5 + p = randcircshift(numout(A), numin(A)) + C = randn!(transpose(A, p)) + Mooncake.TestUtils.test_rule(rng, TensorKit.add_transpose!, C, A, p, α, β; atol, rtol, mode) + A = C + end + end + + @timedtestset "add_braid!" begin + A = randn(T, V[1] ⊗ V[2] ← V[4] ⊗ V[5]) + α = randn(T) + β = randn(T) + + # repeat a couple times to get some distribution of arrows + for _ in 1:5 + p = randcircshift(numout(A), numin(A)) + levels = tuple(randperm(numind(A))) + C = randn!(transpose(A, p)) + Mooncake.TestUtils.test_rule(rng, TensorKit.add_transpose!, C, A, p, α, β; atol, rtol, mode) + A = C + end + end + end + symmetricbraiding && @timedtestset "TensorOperations with scalartype $T" for T in eltypes atol = precision(T) rtol = precision(T) @@ -107,10 +164,10 @@ for V in spacelist p = randindextuple(numind(A)) C1 = randn!(TensorOperations.tensoralloc_add(T, A, p, false, Val(false))) - Mooncake.TestUtils.test_rule(rng, tensoradd!, C1, A, p, false, α, β; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, tensoradd!, C1, A, p, false, α, β; atol, rtol, mode, is_primitive = false) C2 = randn!(TensorOperations.tensoralloc_add(T, A, p, true, Val(false))) - Mooncake.TestUtils.test_rule(rng, tensoradd!, C2, A, p, true, α, β; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, tensoradd!, C2, A, p, true, α, β; atol, rtol, mode, is_primitive = false) A = rand(Bool) ? C1 : C2 end From 2ec198e88730e84f8e1d43222551ea927c23ac0f Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Tue, 20 Jan 2026 12:05:46 -0500 Subject: [PATCH 05/21] add mul rules --- ext/TensorKitMooncakeExt/linalg.jl | 39 ++++++++++++++++++++++++++++++ test/autodiff/mooncake.jl | 18 ++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/ext/TensorKitMooncakeExt/linalg.jl b/ext/TensorKitMooncakeExt/linalg.jl index 56533d227..d0d73d951 100644 --- a/ext/TensorKitMooncakeExt/linalg.jl +++ b/ext/TensorKitMooncakeExt/linalg.jl @@ -1,3 +1,42 @@ +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(mul!), AbstractTensorMap, AbstractTensorMap, AbstractTensorMap, Number, Number} + +function Mooncake.rrule!!( + ::CoDual{typeof(mul!)}, + C_ΔC::CoDual{<:AbstractTensorMap}, A_ΔA::CoDual{<:AbstractTensorMap}, B_ΔB::CoDual{<:AbstractTensorMap}, + α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number} + ) + (C, ΔC), (A, ΔA), (B, ΔB) = arrayify.((C_ΔC, A_ΔA, B_ΔB)) + α, β = primal.((α_Δα, β_Δβ)) + + # primal call + C_cache = copy(C) + AB = if _needs_tangent(α) + AB = A * B + add!(C, AB, α, β) + AB + else + mul!(C, A, B, α, β) + nothing + end + + function mul_pullback(::NoRData) + copy!(C, C_cache) + + scale!(ΔC, conj(β)) + mul!(ΔA, ΔC, B', conj(α), One()) + mul!(ΔB, A', ΔC, conj(α), One()) + ΔCr = NoRData() + ΔAr = NoRData() + ΔBr = NoRData() + Δαr = isnothing(AB) ? NoRData() : Mooncake._rdata(inner(AB, ΔC)) + Δβr = _needs_tangent(β) ? Mooncake._rdata(inner(C, ΔC)) : NoRData() + + return NoRData(), ΔCr, ΔAr, ΔBr, Δαr, Δβr + end + + return C_ΔC, mul_pullback +end + Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(norm), AbstractTensorMap, Real} function Mooncake.rrule!!(::CoDual{typeof(norm)}, tΔt::CoDual{<:AbstractTensorMap}, pdp::CoDual{<:Real}) diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index 2ca21654e..4df18a331 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -102,6 +102,24 @@ for V in spacelist Mooncake.TestUtils.test_rule(rng, inner, C', A'; atol, rtol, mode) end + @timedtestset "LinearAlgebra with scalartype $T" for T in eltypes + atol = precision(T) + rtol = precision(T) + + C = randn(T, V[1] ⊗ V[2] ← V[5]) + A = randn(T, codomain(C) ← V[3] ⊗ V[4]) + B = randn(T, domain(A) ← domain(C)) + α = randn(T) + β = randn(T) + + Mooncake.TestUtils.test_rule(rng, mul!, C, A, B, α, β; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, mul!, C, A, B; atol, rtol, mode, is_primitive = false) + + Mooncake.TestUtils.test_rule(rng, norm, C, 2; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, norm, C', 2; atol, rtol, mode) + end + + @timedtestset "Index manipulations with scalartype $T" for T in eltypes atol = precision(T) rtol = precision(T) From 630e73b29b53b208c0081da225bd328cd120f678 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Tue, 20 Jan 2026 14:56:06 -0500 Subject: [PATCH 06/21] temporarily disable Fibonacci (complex) spaces --- test/autodiff/mooncake.jl | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index 4df18a331..3ca512f56 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -59,13 +59,13 @@ spacelist = ( Vect[SU2Irrep](1 // 2 => 2), Vect[SU2Irrep](0 => 1, 1 // 2 => 1, 3 // 2 => 1)', ), - ( - Vect[FibonacciAnyon](:I => 2, :τ => 1), - Vect[FibonacciAnyon](:I => 1, :τ => 2)', - Vect[FibonacciAnyon](:I => 2, :τ => 2)', - Vect[FibonacciAnyon](:I => 2, :τ => 3), - Vect[FibonacciAnyon](:I => 2, :τ => 2), - ), + # ( + # Vect[FibonacciAnyon](:I => 2, :τ => 1), + # Vect[FibonacciAnyon](:I => 1, :τ => 2)', + # Vect[FibonacciAnyon](:I => 2, :τ => 2)', + # Vect[FibonacciAnyon](:I => 2, :τ => 3), + # Vect[FibonacciAnyon](:I => 2, :τ => 2), + # ), ) for V in spacelist From 06336b6f652d918a3126e1ee32a85f37fdcd9f14 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Tue, 20 Jan 2026 17:10:35 -0500 Subject: [PATCH 07/21] bump TupleTools compat --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 934bdb6ed..11b378c4a 100644 --- a/Project.toml +++ b/Project.toml @@ -56,7 +56,7 @@ TensorKitSectors = "0.3.3" TensorOperations = "5.1" Test = "1" TestExtras = "0.2,0.3" -TupleTools = "1.1" +TupleTools = "1.5" VectorInterface = "0.4.8, 0.5" Zygote = "0.7" cuTENSOR = "2" From 01a4d6e31808b85c8e34a6354a4669064115d3a6 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 14:19:11 -0500 Subject: [PATCH 08/21] add twist! rule --- .../indexmanipulations.jl | 45 +++++++++++++++++++ test/autodiff/mooncake.jl | 8 ++++ 2 files changed, 53 insertions(+) diff --git a/ext/TensorKitMooncakeExt/indexmanipulations.jl b/ext/TensorKitMooncakeExt/indexmanipulations.jl index a0b73dde2..000ae5d83 100644 --- a/ext/TensorKitMooncakeExt/indexmanipulations.jl +++ b/ext/TensorKitMooncakeExt/indexmanipulations.jl @@ -151,3 +151,48 @@ function Mooncake.rrule!!( return C_ΔC, add_braid!_pullback end + +# both are needed for correctly capturing every dispatch +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(twist!), AbstractTensorMap, Any} +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), @NamedTuple{inv::Bool}, typeof(twist!), AbstractTensorMap, Any} + +function Mooncake.rrule!!(::CoDual{typeof(twist!)}, t_Δt::CoDual{<:AbstractTensorMap}, inds_Δinds::CoDual) + # prepare arguments + t, Δt = arrayify(t_Δt) + inv = false + inds = primal(inds_Δinds) + + # primal call + t_cache = copy(t) + twist!(t, inds; inv) + + function twist_pullback(::NoRData) + copy!(t, t_cache) + twist!(Δt, inds; inv = !inv) + return ntuple(Returns(NoRData()), 3) + end + + return t_Δt, twist_pullback + +end +function Mooncake.rrule!!( + ::CoDual{typeof(Core.kwcall)}, kwargs_Δkwargs::CoDual{@NamedTuple{inv::Bool}}, ::CoDual{typeof(twist!)}, + t_Δt::CoDual{<:AbstractTensorMap}, inds_Δinds::CoDual + ) + # prepare arguments + t, Δt = arrayify(t_Δt) + inv = primal(kwargs_Δkwargs).inv + inds = primal(inds_Δinds) + + # primal call + t_cache = copy(t) + twist!(t, inds; inv) + + function twist_pullback(::NoRData) + copy!(t, t_cache) + twist!(Δt, inds; inv = !inv) + return ntuple(Returns(NoRData()), 5) + end + + return t_Δt, twist_pullback +end diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index 3ca512f56..85b251885 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -166,6 +166,14 @@ for V in spacelist A = C end end + + @timedtestset "twist!" begin + A = randn(T, V[1] ⊗ V[2] ← V[4] ⊗ V[5]) + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; inv = false), twist!, A, 1; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; inv = true), twist!, A, [1, 3]; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, twist!, A, 1; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, twist!, A, [1, 3]; atol, rtol, mode) + end end symmetricbraiding && @timedtestset "TensorOperations with scalartype $T" for T in eltypes From ad9238ec7fdfd5050456aa27476b6bb5c9fe4419 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 14:50:48 -0500 Subject: [PATCH 09/21] add flip rule --- .../indexmanipulations.jl | 44 +++++++++++++++++++ test/autodiff/mooncake.jl | 7 ++- 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/ext/TensorKitMooncakeExt/indexmanipulations.jl b/ext/TensorKitMooncakeExt/indexmanipulations.jl index 000ae5d83..9e98023e2 100644 --- a/ext/TensorKitMooncakeExt/indexmanipulations.jl +++ b/ext/TensorKitMooncakeExt/indexmanipulations.jl @@ -196,3 +196,47 @@ function Mooncake.rrule!!( return t_Δt, twist_pullback end + +# both are needed for correctly capturing every dispatch +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(flip), AbstractTensorMap, Any} +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), @NamedTuple{inv::Bool}, typeof(flip), AbstractTensorMap, Any} + +function Mooncake.rrule!!(::CoDual{typeof(flip)}, t_Δt::CoDual{<:AbstractTensorMap}, inds_Δinds::CoDual) + # prepare arguments + t, Δt = arrayify(t_Δt) + inv = false + inds = primal(inds_Δinds) + + # primal call + t_flipped = flip(t, inds; inv) + t_flipped_Δt_flipped = Mooncake.zero_fcodual(t_flipped) + _, Δt_flipped = arrayify(t_flipped_Δt_flipped) + + function twist_pullback(::NoRData) + copy!(Δt, flip(Δt_flipped, inds; inv = !inv)) + return ntuple(Returns(NoRData()), 3) + end + + return t_flipped_Δt_flipped, twist_pullback +end +function Mooncake.rrule!!( + ::CoDual{typeof(Core.kwcall)}, kwargs_Δkwargs::CoDual{@NamedTuple{inv::Bool}}, ::CoDual{typeof(flip)}, + t_Δt::CoDual{<:AbstractTensorMap}, inds_Δinds::CoDual + ) + # prepare arguments + t, Δt = arrayify(t_Δt) + inv = primal(kwargs_Δkwargs).inv + inds = primal(inds_Δinds) + + # primal call + t_flipped = flip(t, inds; inv) + t_flipped_Δt_flipped = Mooncake.zero_fcodual(t_flipped) + _, Δt_flipped = arrayify(t_flipped_Δt_flipped) + + function twist_pullback(::NoRData) + copy!(Δt, flip(Δt_flipped, inds; inv = !inv)) + return ntuple(Returns(NoRData()), 5) + end + + return t_flipped_Δt_flipped, twist_pullback +end diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index 85b251885..ace67dae7 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -167,12 +167,17 @@ for V in spacelist end end - @timedtestset "twist!" begin + @timedtestset "flip_n_twist!" begin A = randn(T, V[1] ⊗ V[2] ← V[4] ⊗ V[5]) Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; inv = false), twist!, A, 1; atol, rtol, mode) Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; inv = true), twist!, A, [1, 3]; atol, rtol, mode) Mooncake.TestUtils.test_rule(rng, twist!, A, 1; atol, rtol, mode) Mooncake.TestUtils.test_rule(rng, twist!, A, [1, 3]; atol, rtol, mode) + + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; inv = false), flip, A, 1; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; inv = true), flip, A, [1, 3]; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, flip, A, 1; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, flip, A, [1, 3]; atol, rtol, mode) end end From 94950d4f62ba6fe1e65a47779726d54b690a4291 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 15:37:21 -0500 Subject: [PATCH 10/21] vector spaces arent vector spaces! --- ext/TensorKitMooncakeExt/utility.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ext/TensorKitMooncakeExt/utility.jl b/ext/TensorKitMooncakeExt/utility.jl index ca2c79b54..f45aaf3bc 100644 --- a/ext/TensorKitMooncakeExt/utility.jl +++ b/ext/TensorKitMooncakeExt/utility.jl @@ -25,4 +25,8 @@ end # Ignore derivatives # ------------------ + +# A VectorSpace has no meaningful notion of a vector space (tangent space) +Mooncake.tangent_type(::Type{<:VectorSpace}) = Mooncake.NoTangent + @zero_derivative DefaultCtx Tuple{typeof(TensorKit.fusionblockstructure), Any} From f8f8529d6c73db02a496724482a2f37ca8e5579c Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 15:50:17 -0500 Subject: [PATCH 11/21] insert and remove units --- .../indexmanipulations.jl | 167 ++++++++++++++++++ test/autodiff/mooncake.jl | 21 +++ 2 files changed, 188 insertions(+) diff --git a/ext/TensorKitMooncakeExt/indexmanipulations.jl b/ext/TensorKitMooncakeExt/indexmanipulations.jl index 9e98023e2..464c18392 100644 --- a/ext/TensorKitMooncakeExt/indexmanipulations.jl +++ b/ext/TensorKitMooncakeExt/indexmanipulations.jl @@ -240,3 +240,170 @@ function Mooncake.rrule!!( return t_flipped_Δt_flipped, twist_pullback end + +for insertunit in (:insertleftunit, :insertrightunit) + insertunit_pullback = Symbol(insertunit, :_pullback) + @eval begin + # both are needed for correctly capturing every dispatch + Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof($insertunit), AbstractTensorMap, Val} + Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), NamedTuple, typeof($insertunit), AbstractTensorMap, Val} + + function Mooncake.rrule!!(::CoDual{typeof($insertunit)}, tsrc_Δtsrc::CoDual{<:AbstractTensorMap}, ival_Δival::CoDual{<:Val}) + # prepare arguments + tsrc, Δtsrc = arrayify(tsrc_Δtsrc) + ival = primal(ival_Δival) + + # tdst shares data with tsrc if <:TensorMap, in this case we have to deal with correctly + # sharing address spaces + if tsrc isa TensorMap + tsrc_cache = copy(tsrc) + tdst = $insertunit(tsrc, ival) + # note: this is somewhat of a hack that makes use of the fact that the tangent is + # encoded without any information about the space, which allows us to simply reuse + # the tangent exactly without having to modify the space information + tdst_Δtdst = CoDual(tdst, Mooncake.tangent(tsrc_Δtsrc)) + else + tsrc_cache = nothing + tdst = $insertunit(tsrc, ival) + tdst_Δtdst = Mooncake.zero_fcodual(tdst) + end + + _, Δtdst = arrayify(tdst_Δtdst) + + function $insertunit_pullback(::NoRData) + if isnothing(tsrc_cache) + for (c, b) in blocks(Δtdst) + copy!(block(Δtsrc, c), b) + end + else + copy!(tsrc, tsrc_cache) + # note: since data is already shared, don't have to do anything here! + end + return ntuple(Returns(NoRData()), 3) + end + + return tdst_Δtdst, $insertunit_pullback + end + function Mooncake.rrule!!( + ::CoDual{typeof(Core.kwcall)}, kwargs_Δkwargs::CoDual{<:NamedTuple}, + ::CoDual{typeof($insertunit)}, tsrc_Δtsrc::CoDual{<:AbstractTensorMap}, ival_Δival::CoDual{<:Val} + ) + # prepare arguments + tsrc, Δtsrc = arrayify(tsrc_Δtsrc) + ival = primal(ival_Δival) + kwargs = primal(kwargs_Δkwargs) + + # tdst shares data with tsrc if <:TensorMap & copy=false, in this case we have to deal with correctly + # sharing address spaces + if tsrc isa TensorMap && !get(kwargs, :copy, false) + tsrc_cache = copy(tsrc) + tdst = $insertunit(tsrc, ival; kwargs...) + # note: this is somewhat of a hack that makes use of the fact that the tangent is + # encoded without any information about the space, which allows us to simply reuse + # the tangent exactly without having to modify the space information + tdst_Δtdst = CoDual(tdst, Mooncake.tangent(tsrc_Δtsrc)) + else + tsrc_cache = nothing + tdst = $insertunit(tsrc, ival; kwargs...) + tdst_Δtdst = Mooncake.zero_fcodual(tdst) + end + + _, Δtdst = arrayify(tdst_Δtdst) + + function $insertunit_pullback(::NoRData) + if isnothing(tsrc_cache) + for (c, b) in blocks(Δtdst) + copy!(block(Δtsrc, c), b) + end + else + copy!(tsrc, tsrc_cache) + # note: since data is already shared, don't have to do anything here! + end + return ntuple(Returns(NoRData()), 5) + end + + return tdst_Δtdst, $insertunit_pullback + end + end +end + + +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(removeunit), AbstractTensorMap, Val} +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), NamedTuple, typeof(removeunit), AbstractTensorMap, Val} + +function Mooncake.rrule!!(::CoDual{typeof(removeunit)}, tsrc_Δtsrc::CoDual{<:AbstractTensorMap}, ival_Δival::CoDual{Val{i}}) where {i} + # prepare arguments + tsrc, Δtsrc = arrayify(tsrc_Δtsrc) + ival = primal(ival_Δival) + + # tdst shares data with tsrc if <:TensorMap, in this case we have to deal with correctly + # sharing address spaces + if tsrc isa TensorMap + tsrc_cache = copy(tsrc) + tdst = removeunit(tsrc, ival) + # note: this is somewhat of a hack that makes use of the fact that the tangent is + # encoded without any information about the space, which allows us to simply reuse + # the tangent exactly without having to modify the space information + tdst_Δtdst = CoDual(tdst, Mooncake.tangent(tsrc_Δtsrc)) + else + tsrc_cache = nothing + tdst = removeunit(tsrc, ival) + tdst_Δtdst = Mooncake.zero_fcodual(tdst) + end + + _, Δtdst = arrayify(tdst_Δtdst) + + function removeunit_pullback(::NoRData) + if isnothing(tsrc_cache) + for (c, b) in blocks(Δtdst) + copy!(block(Δtsrc, c), b) + end + else + copy!(tsrc, tsrc_cache) + # note: since data is already shared, don't have to do anything here! + end + return ntuple(Returns(NoRData()), 3) + end + + return tdst_Δtdst, removeunit_pullback +end +function Mooncake.rrule!!( + ::CoDual{typeof(Core.kwcall)}, kwargs_Δkwargs::CoDual{<:NamedTuple}, + ::CoDual{typeof(removeunit)}, tsrc_Δtsrc::CoDual{<:AbstractTensorMap}, ival_Δival::CoDual{<:Val} + ) + # prepare arguments + tsrc, Δtsrc = arrayify(tsrc_Δtsrc) + ival = primal(ival_Δival) + kwargs = primal(kwargs_Δkwargs) + + # tdst shares data with tsrc if <:TensorMap & copy=false, in this case we have to deal with correctly + # sharing address spaces + if tsrc isa TensorMap && !get(kwargs, :copy, false) + tsrc_cache = copy(tsrc) + tdst = removeunit(tsrc, ival; kwargs...) + # note: this is somewhat of a hack that makes use of the fact that the tangent is + # encoded without any information about the space, which allows us to simply reuse + # the tangent exactly without having to modify the space information + tdst_Δtdst = CoDual(tdst, Mooncake.tangent(tsrc_Δtsrc)) + else + tsrc_cache = nothing + tdst = removeunit(tsrc, ival; kwargs...) + tdst_Δtdst = Mooncake.zero_fcodual(tdst) + end + + _, Δtdst = arrayify(tdst_Δtdst) + + function removeunit_pullback(::NoRData) + if isnothing(tsrc_cache) + for (c, b) in blocks(Δtdst) + copy!(block(Δtsrc, c), b) + end + else + copy!(tsrc, tsrc_cache) + # note: since data is already shared, don't have to do anything here! + end + return ntuple(Returns(NoRData()), 5) + end + + return tdst_Δtdst, removeunit_pullback +end diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index ace67dae7..a5b08fc90 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -179,6 +179,27 @@ for V in spacelist Mooncake.TestUtils.test_rule(rng, flip, A, 1; atol, rtol, mode) Mooncake.TestUtils.test_rule(rng, flip, A, [1, 3]; atol, rtol, mode) end + + @timedtestset "insert and remove units" begin + A = randn(T, V[1] ⊗ V[2] ← V[4] ⊗ V[5]) + + for insertunit in (insertleftunit, insertrightunit) + Mooncake.TestUtils.test_rule(rng, insertunit, A, Val(1); atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, insertunit, A, Val(4); atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, insertunit, A', Val(2); atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; copy = false), insertunit, A, Val(1); atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; copy = true), insertunit, A, Val(2); atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; copy = false, dual = true, conj = true), insertunit, A, Val(3); atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; copy = false, dual = true, conj = true), insertunit, A', Val(3); atol, rtol, mode) + end + + for i in 1:4 + B = insertleftunit(A, i; dual = rand(Bool)) + Mooncake.TestUtils.test_rule(rng, removeunit, B, Val(i); atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; copy = false), removeunit, B, Val(i); atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, Core.kwcall, (; copy = true), removeunit, B, Val(i); atol, rtol, mode) + end + end end symmetricbraiding && @timedtestset "TensorOperations with scalartype $T" for T in eltypes From ab92b46e149d00083917cc74516e81be462db1a7 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 17:08:30 -0500 Subject: [PATCH 12/21] mark a bunch of things as non-differentiable --- ext/TensorKitMooncakeExt/utility.jl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ext/TensorKitMooncakeExt/utility.jl b/ext/TensorKitMooncakeExt/utility.jl index f45aaf3bc..e93de22be 100644 --- a/ext/TensorKitMooncakeExt/utility.jl +++ b/ext/TensorKitMooncakeExt/utility.jl @@ -30,3 +30,10 @@ end Mooncake.tangent_type(::Type{<:VectorSpace}) = Mooncake.NoTangent @zero_derivative DefaultCtx Tuple{typeof(TensorKit.fusionblockstructure), Any} + +@zero_derivative DefaultCtx Tuple{typeof(TensorKit.select), HomSpace, Index2Tuple} +@zero_derivative DefaultCtx Tuple{typeof(TensorKit.flip), HomSpace, Any} +@zero_derivative DefaultCtx Tuple{typeof(TensorKit.permute), HomSpace, Index2Tuple} +@zero_derivative DefaultCtx Tuple{typeof(TensorKit.braid), HomSpace, Index2Tuple, IndexTuple} +@zero_derivative DefaultCtx Tuple{typeof(TensorKit.compose), HomSpace, HomSpace} +@zero_derivative DefaultCtx Tuple{typeof(TensorOperations.tensorcontract), HomSpace, Index2Tuple, Bool, HomSpace, Index2Tuple, Bool, Index2Tuple} From cfc301e8e87bcaf8ae7c5ed64e3b7b698a3aa8bf Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 17:08:53 -0500 Subject: [PATCH 13/21] rewrite rule for `tensortrace!` in terms of `trace_permute!` --- ext/TensorKitMooncakeExt/tensoroperations.jl | 53 +++++++++----------- test/autodiff/mooncake.jl | 16 +++--- 2 files changed, 32 insertions(+), 37 deletions(-) diff --git a/ext/TensorKitMooncakeExt/tensoroperations.jl b/ext/TensorKitMooncakeExt/tensoroperations.jl index 915a10356..989ee2830 100644 --- a/ext/TensorKitMooncakeExt/tensoroperations.jl +++ b/ext/TensorKitMooncakeExt/tensoroperations.jl @@ -230,83 +230,80 @@ Mooncake.@is_primitive( DefaultCtx, ReverseMode, Tuple{ - typeof(TO.tensortrace!), + typeof(TensorKit.trace_permute!), AbstractTensorMap, - AbstractTensorMap, Index2Tuple, Index2Tuple, Bool, + AbstractTensorMap, Index2Tuple, Index2Tuple, Number, Number, - Vararg{Any}, + Any, } ) function Mooncake.rrule!!( - ::CoDual{typeof(TO.tensortrace!)}, + ::CoDual{typeof(TensorKit.trace_permute!)}, C_ΔC::CoDual{<:AbstractTensorMap}, - A_ΔA::CoDual{<:AbstractTensorMap}, p_Δp::CoDual{<:Index2Tuple}, q_Δq::CoDual{<:Index2Tuple}, conjA_ΔconjA::CoDual{Bool}, + A_ΔA::CoDual{<:AbstractTensorMap}, p_Δp::CoDual{<:Index2Tuple}, q_Δq::CoDual{<:Index2Tuple}, α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, - ba_Δba::CoDual... + backend_Δbackend::CoDual ) # prepare arguments C, ΔC = arrayify(C_ΔC) A, ΔA = arrayify(A_ΔA) p = primal(p_Δp) q = primal(q_Δq) - conjA = primal(conjA_ΔconjA) α, β = primal.((α_Δα, β_Δβ)) - ba = primal.(ba_Δba) + backend = primal(backend_Δbackend) # primal call C_cache = copy(C) - TO.tensortrace!(C, A, p, q, conjA, α, β, ba...) + TensorKit.trace_permute!(C, A, p, q, α, β, backend) - function tensortrace_pullback(::NoRData) + function trace_permute_pullback(::NoRData) copy!(C, C_cache) - ΔCr = tensortrace_pullback_ΔC!(ΔC, β) - ΔAr = tensortrace_pullback_ΔA!(ΔA, ΔC, A, p, q, conjA, α, ba...) - Δαr = tensortrace_pullback_Δα(ΔC, A, p, q, conjA, α, ba...) - Δβr = tensortrace_pullback_Δβ(ΔC, C, β) + ΔAr = trace_permute_pullback_ΔA!(ΔA, ΔC, A, p, q, α, backend) + Δαr = trace_permute_pullback_Δα(ΔC, A, p, q, α, backend) + Δβr = trace_permute_pullback_Δβ(ΔC, C, β) + ΔCr = trace_permute_pullback_ΔC!(ΔC, β) return NoRData(), - ΔCr, - ΔAr, NoRData(), NoRData(), NoRData(), - Δαr, Δβr, - map(Returns(NoRData()), ba)... + ΔCr, ΔAr, NoRData(), NoRData(), + Δαr, Δβr, NoRData() end - return C_ΔC, tensortrace_pullback + return C_ΔC, trace_permute_pullback end -tensortrace_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) +trace_permute_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) -function tensortrace_pullback_ΔA!( - ΔA, ΔC, A, p, q, conjA, α, ba... +function trace_permute_pullback_ΔA!( + ΔA, ΔC, A, p, q, α, backend ) ip = invperm((linearize(p)..., q[1]..., q[2]...)) pdA = _repartition(ip, A) - E = one!(TO.tensoralloc_add(scalartype(A), A, q, conjA)) + E = one!(TO.tensoralloc_add(scalartype(A), A, q, false)) twist!(E, filter(x -> !isdual(space(E, x)), codomainind(E))) pE = ((), trivtuple(TO.numind(q))) pΔC = (trivtuple(TO.numind(p)), ()) TO.tensorproduct!( - ΔA, ΔC, pΔC, conjA, E, pE, conjA, pdA, conjA ? α : conj(α), Zero(), ba... + ΔA, ΔC, pΔC, false, E, pE, false, pdA, conj(α), One(), backend ) return NoRData() end -function tensortrace_pullback_Δα( - ΔC, A, p, q, conjA, α, ba... +function trace_permute_pullback_Δα( + ΔC, A, p, q, α, backend ) Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) Tdα === NoRData && return NoRData() # TODO: this result might be easier to compute as: # C′ = βC + α * trace(A) ⟹ At = (C′ - βC) / α - At = TO.tensortrace(A, p, q, conjA) + At = TO.tensortrace(A, p, q, false, One(), backend) Δα = inner(At, ΔC) return Mooncake._rdata(Δα) end -function tensortrace_pullback_Δβ(ΔC, C, β) +function trace_permute_pullback_Δβ(ΔC, C, β) Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) Tdβ === NoRData && return NoRData() diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index a5b08fc90..cca2c92d3 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -260,14 +260,14 @@ for V in spacelist ) Mooncake.TestUtils.test_rule( rng, tensorcontract!, C, A, pA, conjA, B, pB, conjB, pAB, α, β; - atol, rtol, mode, is_primitive + atol, rtol, mode ) end end end - @timedtestset "tensortrace!" begin + @timedtestset "trace_permute!" begin for _ in 1:5 k1 = rand(0:2) k2 = rand(1:2) @@ -282,13 +282,11 @@ for V in spacelist α = randn(T) β = randn(T) - for conjA in (false, true) - C = randn!(TensorOperations.tensoralloc_add(T, A, p, conjA, Val(false))) - Mooncake.TestUtils.test_rule( - rng, tensortrace!, C, A, p, q, conjA, α, β; - atol, rtol, mode, is_primitive = false - ) - end + C = randn!(TensorOperations.tensoralloc_add(T, A, p, false, Val(false))) + Mooncake.TestUtils.test_rule( + rng, TensorKit.trace_permute!, C, A, p, q, α, β, TensorOperations.DefaultBackend(); + atol, rtol, mode + ) end end end From f566acb8646193c973ac2448d16d213ef69f11e2 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 17:10:38 -0500 Subject: [PATCH 14/21] dont need rules for `tensoradd!` --- ext/TensorKitMooncakeExt/tensoroperations.jl | 86 -------------------- test/autodiff/mooncake.jl | 19 ----- 2 files changed, 105 deletions(-) diff --git a/ext/TensorKitMooncakeExt/tensoroperations.jl b/ext/TensorKitMooncakeExt/tensoroperations.jl index 989ee2830..7b979d4cf 100644 --- a/ext/TensorKitMooncakeExt/tensoroperations.jl +++ b/ext/TensorKitMooncakeExt/tensoroperations.jl @@ -1,89 +1,3 @@ -# tensoradd! -# ---------- -# Mooncake.@is_primitive( -# DefaultCtx, -# ReverseMode, -# Tuple{ -# typeof(TO.tensoradd!), -# AbstractTensorMap, -# AbstractTensorMap, Index2Tuple, Bool, -# Number, Number, Vararg{Any}, -# } -# ) -# -# function Mooncake.rrule!!( -# ::CoDual{typeof(TO.tensoradd!)}, -# C_ΔC::CoDual{<:AbstractTensorMap}, -# A_ΔA::CoDual{<:AbstractTensorMap}, pA_ΔpA::CoDual{<:Index2Tuple}, conjA_ΔconjA::CoDual{Bool}, -# α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, -# ba_Δba::CoDual... -# ) -# # prepare arguments -# C, ΔC = arrayify(C_ΔC) -# A, ΔA = arrayify(A_ΔA) -# pA = primal(pA_ΔpA) -# conjA = primal(conjA_ΔconjA) -# α, β = primal.((α_Δα, β_Δβ)) -# ba = primal.(ba_Δba) -# -# # primal call -# C_cache = copy(C) -# TO.tensoradd!(C, A, pA, conjA, α, β, ba...) -# -# function tensoradd_pullback(::NoRData) -# copy!(C, C_cache) -# -# ΔCr = tensoradd_pullback_ΔC!(ΔC, β) -# ΔAr = tensoradd_pullback_ΔA!(ΔA, ΔC, A, pA, conjA, α, ba...) -# Δαr = tensoradd_pullback_Δα(ΔC, A, pA, conjA, α, ba...) -# Δβr = tensoradd_pullback_Δβ(ΔC, C, β) -# -# return NoRData(), -# ΔCr, -# ΔAr, NoRData(), NoRData(), -# Δαr, Δβr, -# map(Returns(NoRData()), ba)... -# end -# -# return C_ΔC, tensoradd_pullback -# end -# -# tensoradd_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) -# -# function tensoradd_pullback_ΔA!( -# ΔA, ΔC, A, pA, conjA, α, ba... -# ) -# ipA = invperm(linearize(pA)) -# pΔA = _repartition(ipA, A) -# TO.tensoradd!(ΔA, ΔC, pΔA, conjA, conjA ? α : conj(α), Zero(), ba...) -# return NoRData() -# end -# -# function tensoradd_pullback_Δα( -# ΔC, A, pA, conjA, α, ba... -# ) -# Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) -# Tdα === NoRData && return NoRData() -# -# tΔC = twist(ΔC, filter(x -> isdual(space(ΔC, x)), allind(ΔC)); copy = false) -# Δα = TO.tensorscalar( -# TO.tensorcontract( -# A, ((), linearize(pA)), !conjA, -# tΔC, (trivtuple(TO.numind(pA)), ()), false, -# ((), ()), One(), ba... -# ) -# ) -# return Mooncake._rdata(Δα) -# end -# -# function tensoradd_pullback_Δβ(ΔC, C, β) -# Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) -# Tdβ === NoRData && return NoRData() -# -# Δβ = inner(C, ΔC) -# return Mooncake._rdata(Δβ) -# end - # tensorcontract! # --------------- Mooncake.@is_primitive( diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index cca2c92d3..066a3585f 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -206,25 +206,6 @@ for V in spacelist atol = precision(T) rtol = precision(T) - @timedtestset "tensoradd!" begin - A = randn(T, V[1] ⊗ V[2] ← V[4] ⊗ V[5]) - α = randn(T) - β = randn(T) - - # repeat a couple times to get some distribution of arrows - for _ in 1:5 - p = randindextuple(numind(A)) - - C1 = randn!(TensorOperations.tensoralloc_add(T, A, p, false, Val(false))) - Mooncake.TestUtils.test_rule(rng, tensoradd!, C1, A, p, false, α, β; atol, rtol, mode, is_primitive = false) - - C2 = randn!(TensorOperations.tensoralloc_add(T, A, p, true, Val(false))) - Mooncake.TestUtils.test_rule(rng, tensoradd!, C2, A, p, true, α, β; atol, rtol, mode, is_primitive = false) - - A = rand(Bool) ? C1 : C2 - end - end - @timedtestset "tensorcontract!" begin for _ in 1:5 d = 0 From 13c3779295268a2175eda6a5b984b4aa35e9964a Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 19:52:14 -0500 Subject: [PATCH 15/21] add planaroperations --- .../TensorKitMooncakeExt.jl | 1 + ext/TensorKitMooncakeExt/planaroperations.jl | 88 +++++++++++++++++++ test/autodiff/mooncake.jl | 73 +++++++++++++++ 3 files changed, 162 insertions(+) create mode 100644 ext/TensorKitMooncakeExt/planaroperations.jl diff --git a/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl b/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl index 15e0c4c9f..4c692adb9 100644 --- a/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl +++ b/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl @@ -15,5 +15,6 @@ include("linalg.jl") include("indexmanipulations.jl") include("vectorinterface.jl") include("tensoroperations.jl") +include("planaroperations.jl") end diff --git a/ext/TensorKitMooncakeExt/planaroperations.jl b/ext/TensorKitMooncakeExt/planaroperations.jl new file mode 100644 index 000000000..a480293af --- /dev/null +++ b/ext/TensorKitMooncakeExt/planaroperations.jl @@ -0,0 +1,88 @@ +# planartrace! +# ------------ +Mooncake.@is_primitive( + DefaultCtx, + ReverseMode, + Tuple{ + typeof(TensorKit.planartrace!), + AbstractTensorMap, + AbstractTensorMap, Index2Tuple, Index2Tuple, + Number, Number, + Any, Any, + } +) + +function Mooncake.rrule!!( + ::CoDual{typeof(TensorKit.planartrace!)}, + C_ΔC::CoDual{<:AbstractTensorMap}, + A_ΔA::CoDual{<:AbstractTensorMap}, p_Δp::CoDual{<:Index2Tuple}, q_Δq::CoDual{<:Index2Tuple}, + α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, + backend_Δbackend::CoDual, allocator_Δallocator::CoDual + ) + # prepare arguments + C, ΔC = arrayify(C_ΔC) + A, ΔA = arrayify(A_ΔA) + p = primal(p_Δp) + q = primal(q_Δq) + α, β = primal.((α_Δα, β_Δβ)) + backend, allocator = primal.((backend_Δbackend, allocator_Δallocator)) + + # primal call + C_cache = copy(C) + TensorKit.planartrace!(C, A, p, q, α, β, backend, allocator) + + function planartrace_pullback(::NoRData) + copy!(C, C_cache) + + ΔAr = planartrace_pullback_ΔA!(ΔA, ΔC, A, p, q, α, backend, allocator) + Δαr = planartrace_pullback_Δα(ΔC, A, p, q, α, backend, allocator) + Δβr = planartrace_pullback_Δβ(ΔC, C, β) + ΔCr = planartrace_pullback_ΔC!(ΔC, β) + + return NoRData(), + ΔCr, ΔAr, NoRData(), NoRData(), + Δαr, Δβr, NoRData(), NoRData() + end + + return C_ΔC, planartrace_pullback +end + +planartrace_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) + +function planartrace_pullback_ΔA!( + ΔA, ΔC, A, p, q, α, backend, allocator + ) + ip = invperm((linearize(p)..., q[1]..., q[2]...)) + pdA = _repartition(ip, A) + E = one!(TO.tensoralloc_add(scalartype(A), A, q, false)) + twist!(E, filter(x -> !isdual(space(E, x)), codomainind(E))) + pE = ((), trivtuple(TO.numind(q))) + pΔC = (trivtuple(TO.numind(p)), ()) + TensorKit.planarcontract!( + ΔA, ΔC, pΔC, E, pE, pdA, conj(α), One(), backend, allocator + ) + return NoRData() +end + +function planartrace_pullback_Δα( + ΔC, A, p, q, α, backend, allocator + ) + Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) + Tdα === NoRData && return NoRData() + + # TODO: this result might be easier to compute as: + # C′ = βC + α * trace(A) ⟹ At = (C′ - βC) / α + At = TO.tensoralloc_add(scalartype(A), A, p, false, Val(true), allocator) + TensorKit.planartrace!(At, A, p, q, false, One(), backend, allocator) + Δα = inner(At, ΔC) + TO.tensorfree!(At, allocator) + return Mooncake._rdata(Δα) +end + +function planartrace_pullback_Δβ(ΔC, C, β) + Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) + Tdβ === NoRData && return NoRData() + + Δβ = inner(C, ΔC) + return Mooncake._rdata(Δβ) +end diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index 066a3585f..14aaea251 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -271,4 +271,77 @@ for V in spacelist end end end + + @timedtestset "PlanarOperations with scalartype $T" for T in eltypes + atol = precision(T) + rtol = precision(T) + + @timedtestset "planarcontract!" begin + for _ in 1:5 + d = 0 + local V1, V2, V3, k1, k2, k3 + # retry a couple times to make sure there are at least some nonzero elements + for _ in 1:10 + k1 = rand(0:3) + k2 = rand(0:2) + k3 = rand(0:2) + V1 = prod(v -> rand(Bool) ? v' : v, rand(V, k1); init = one(V[1])) + V2 = prod(v -> rand(Bool) ? v' : v, rand(V, k2); init = one(V[1])) + V3 = prod(v -> rand(Bool) ? v' : v, rand(V, k3); init = one(V[1])) + d = min(dim(V1 ← V2), dim(V1' ← V2), dim(V2 ← V3), dim(V2' ← V3)) + d > 1 && break + end + k′ = rand(0:(k1 + k2)) + pA = randcircshift(k′, k1 + k2 - k′, k1) + ipA = _repartition(invperm(linearize(pA)), k′) + k′ = rand(0:(k2 + k3)) + pB = randcircshift(k′, k2 + k3 - k′, k2) + ipB = _repartition(invperm(linearize(pB)), k′) + # TODO: primal value already is broken for this? + # pAB = randcircshift(k1, k3) + pAB = _repartition(tuple((1:(k1 + k3))...), k1) + + α = randn(T) + β = randn(T) + + A = randn(T, permute(V1 ← V2, ipA)) + B = randn(T, permute(V2 ← V3, ipB)) + C = randn!( + TensorOperations.tensoralloc_contract( + T, A, pA, false, B, pB, false, pAB, Val(false) + ) + ) + Mooncake.TestUtils.test_rule( + rng, TensorKit.planarcontract!, C, A, pA, B, pB, pAB, α, β; + atol, rtol, mode, is_primitive = false + ) + end + end + + @timedtestset "planartrace!" begin + for _ in 1:5 + k1 = rand(0:2) + k2 = rand(1:2) + V1 = map(v -> rand(Bool) ? v' : v, rand(V, k1)) + V2 = map(v -> rand(Bool) ? v' : v, rand(V, k2)) + + k′ = rand(0:(k1 + 2k2)) + (_p, _q) = randcircshift(k′, k1 + 2 * k2 - k′, k1) + p = _repartition(_p, rand(0:k1)) + q = _repartition(_q, k2) + ip = _repartition(invperm(linearize((_p, _q))), k′) + A = randn(T, permute(prod(V1) ⊗ prod(V2) ← prod(V2), ip)) + + α = randn(T) + β = randn(T) + C = randn!(TensorOperations.tensoralloc_add(T, A, p, false, Val(false))) + Mooncake.TestUtils.test_rule( + rng, TensorKit.planartrace!, + C, A, p, q, α, β, + TensorOperations.DefaultBackend(), TensorOperations.DefaultAllocator(); + atol, rtol, mode + ) + end + end + end end From 7a6adcc093e718d1d02c784fecd3658e97706b54 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 19:52:39 -0500 Subject: [PATCH 16/21] rewrite rule `tensorcontract` in terms of `blas_contract!` --- ext/TensorKitMooncakeExt/tensoroperations.jl | 83 +++++++++----------- test/autodiff/mooncake.jl | 25 +++--- 2 files changed, 51 insertions(+), 57 deletions(-) diff --git a/ext/TensorKitMooncakeExt/tensoroperations.jl b/ext/TensorKitMooncakeExt/tensoroperations.jl index 7b979d4cf..59a398e27 100644 --- a/ext/TensorKitMooncakeExt/tensoroperations.jl +++ b/ext/TensorKitMooncakeExt/tensoroperations.jl @@ -4,72 +4,69 @@ Mooncake.@is_primitive( DefaultCtx, ReverseMode, Tuple{ - typeof(TO.tensorcontract!), + typeof(TensorKit.blas_contract!), AbstractTensorMap, - AbstractTensorMap, Index2Tuple, Bool, - AbstractTensorMap, Index2Tuple, Bool, + AbstractTensorMap, Index2Tuple, + AbstractTensorMap, Index2Tuple, Index2Tuple, Number, Number, - Vararg{Any}, + Any, Any, } ) function Mooncake.rrule!!( - ::CoDual{typeof(TO.tensorcontract!)}, + ::CoDual{typeof(TensorKit.blas_contract!)}, C_ΔC::CoDual{<:AbstractTensorMap}, - A_ΔA::CoDual{<:AbstractTensorMap}, pA_ΔpA::CoDual{<:Index2Tuple}, conjA_ΔconjA::CoDual{Bool}, - B_ΔB::CoDual{<:AbstractTensorMap}, pB_ΔpB::CoDual{<:Index2Tuple}, conjB_ΔconjB::CoDual{Bool}, + A_ΔA::CoDual{<:AbstractTensorMap}, pA_ΔpA::CoDual{<:Index2Tuple}, + B_ΔB::CoDual{<:AbstractTensorMap}, pB_ΔpB::CoDual{<:Index2Tuple}, pAB_ΔpAB::CoDual{<:Index2Tuple}, α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}, - ba_Δba::CoDual..., + backend_Δbackend::CoDual, allocator_Δallocator::CoDual ) # prepare arguments (C, ΔC), (A, ΔA), (B, ΔB) = arrayify.((C_ΔC, A_ΔA, B_ΔB)) pA, pB, pAB = primal.((pA_ΔpA, pB_ΔpB, pAB_ΔpAB)) - conjA, conjB = primal.((conjA_ΔconjA, conjB_ΔconjB)) α, β = primal.((α_Δα, β_Δβ)) - ba = primal.(ba_Δba) + backend, allocator = primal.((backend_Δbackend, allocator_Δallocator)) # primal call C_cache = copy(C) - TO.tensorcontract!(C, A, pA, conjA, B, pB, conjB, pAB, α, β, ba...) + TensorKit.blas_contract!(C, A, pA, B, pB, pAB, α, β, backend, allocator) - function tensorcontract_pullback(::NoRData) + function blas_contract_pullback(::NoRData) copy!(C, C_cache) - ΔCr = tensorcontract_pullback_ΔC!(ΔC, β) - ΔAr = tensorcontract_pullback_ΔA!( - ΔA, ΔC, A, pA, conjA, B, pB, conjB, pAB, α, ba... + ΔAr = blas_contract_pullback_ΔA!( + ΔA, ΔC, A, pA, B, pB, pAB, α, backend, allocator ) - ΔBr = tensorcontract_pullback_ΔB!( - ΔB, ΔC, A, pA, conjA, B, pB, conjB, pAB, α, ba... + ΔBr = blas_contract_pullback_ΔB!( + ΔB, ΔC, A, pA, B, pB, pAB, α, backend, allocator ) - Δαr = tensorcontract_pullback_Δα( - ΔC, A, pA, conjA, B, pB, conjB, pAB, α, ba... + Δαr = blas_contract_pullback_Δα( + ΔC, A, pA, B, pB, pAB, α, backend, allocator ) - Δβr = tensorcontract_pullback_Δβ(ΔC, C, β) + Δβr = blas_contract_pullback_Δβ(ΔC, C, β) + ΔCr = blas_contract_pullback_ΔC!(ΔC, β) return NoRData(), ΔCr, - ΔAr, NoRData(), NoRData(), - ΔBr, NoRData(), NoRData(), + ΔAr, NoRData(), + ΔBr, NoRData(), NoRData(), Δαr, Δβr, - map(ba_ -> NoRData(), ba)... + NoRData(), NoRData() end - return C_ΔC, tensorcontract_pullback + return C_ΔC, blas_contract_pullback end -tensorcontract_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) +blas_contract_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) -function tensorcontract_pullback_ΔA!( - ΔA, ΔC, A, pA, conjA, B, pB, conjB, pAB, α, ba... +function blas_contract_pullback_ΔA!( + ΔA, ΔC, A, pA, B, pB, pAB, α, backend, allocator ) ipAB = invperm(linearize(pAB)) pΔC = _repartition(ipAB, TO.numout(pA)) ipA = _repartition(invperm(linearize(pA)), A) - conjΔC = conjA - conjB′ = conjA ? conjB : !conjB tB = twist( B, @@ -81,24 +78,22 @@ function tensorcontract_pullback_ΔA!( TO.tensorcontract!( ΔA, - ΔC, pΔC, conjΔC, - tB, reverse(pB), conjB′, + ΔC, pΔC, false, + tB, reverse(pB), true, ipA, - conjA ? α : conj(α), Zero(), - ba... + conj(α), Zero(), + backend, allocator ) return NoRData() end -function tensorcontract_pullback_ΔB!( - ΔB, ΔC, A, pA, conjA, B, pB, conjB, pAB, α, ba... +function blas_contract_pullback_ΔB!( + ΔB, ΔC, A, pA, B, pB, pAB, α, backend, allocator ) ipAB = invperm(linearize(pAB)) pΔC = _repartition(ipAB, TO.numout(pA)) ipB = _repartition(invperm(linearize(pB)), B) - conjΔC = conjB - conjA′ = conjB ? conjA : !conjA tA = twist( A, @@ -110,27 +105,27 @@ function tensorcontract_pullback_ΔB!( TO.tensorcontract!( ΔB, - tA, reverse(pA), conjA′, - ΔC, pΔC, conjΔC, + tA, reverse(pA), true, + ΔC, pΔC, false, ipB, - conjB ? α : conj(α), Zero(), ba... + conj(α), Zero(), backend, allocator ) return NoRData() end -function tensorcontract_pullback_Δα( - ΔC, A, pA, conjA, B, pB, conjB, pAB, α, ba... +function blas_contract_pullback_Δα( + ΔC, A, pA, B, pB, pAB, α, backend, allocator ) Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) Tdα === NoRData && return NoRData() - AB = TO.tensorcontract(A, pA, conjA, B, pB, conjB, pAB, One(), ba...) + AB = TO.tensorcontract(A, pA, false, B, pB, false, pAB, One(), backend, allocator) Δα = inner(AB, ΔC) return Mooncake._rdata(Δα) end -function tensorcontract_pullback_Δβ(ΔC, C, β) +function blas_contract_pullback_Δβ(ΔC, C, β) Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) Tdβ === NoRData && return NoRData() diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index 14aaea251..a1bce9906 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -231,20 +231,19 @@ for V in spacelist β = randn(T) V2_conj = prod(conj, V2; init = one(V[1])) - for conjA in (false, true), conjB in (false, true) - A = randn(T, permute(V1 ← (conjA ? V2_conj : V2), ipA)) - B = randn(T, permute((conjB ? V2_conj : V2) ← V3, ipB)) - C = randn!( - TensorOperations.tensoralloc_contract( - T, A, pA, conjA, B, pB, conjB, pAB, Val(false) - ) - ) - Mooncake.TestUtils.test_rule( - rng, tensorcontract!, C, A, pA, conjA, B, pB, conjB, pAB, α, β; - atol, rtol, mode + A = randn(T, permute(V1 ← V2, ipA)) + B = randn(T, permute(V2 ← V3, ipB)) + C = randn!( + TensorOperations.tensoralloc_contract( + T, A, pA, false, B, pB, false, pAB, Val(false) ) - - end + ) + Mooncake.TestUtils.test_rule( + rng, TensorKit.blas_contract!, + C, A, pA, B, pB, pAB, α, β, + TensorOperations.DefaultBackend(), TensorOperations.DefaultAllocator(); + atol, rtol, mode + ) end end From 4a4b09bf63fba768379483319ef68a4f7e8dddd2 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Wed, 21 Jan 2026 20:50:06 -0500 Subject: [PATCH 17/21] add rule `tr` --- ext/TensorKitMooncakeExt/linalg.jl | 16 ++++++++++++++++ test/autodiff/mooncake.jl | 8 ++++++++ 2 files changed, 24 insertions(+) diff --git a/ext/TensorKitMooncakeExt/linalg.jl b/ext/TensorKitMooncakeExt/linalg.jl index d0d73d951..092ddf369 100644 --- a/ext/TensorKitMooncakeExt/linalg.jl +++ b/ext/TensorKitMooncakeExt/linalg.jl @@ -51,3 +51,19 @@ function Mooncake.rrule!!(::CoDual{typeof(norm)}, tΔt::CoDual{<:AbstractTensorM end return CoDual(n, Mooncake.NoFData()), norm_pullback end + +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(tr), AbstractTensorMap} + +function Mooncake.rrule!!(::CoDual{typeof(tr)}, A_ΔA::CoDual{<:AbstractTensorMap}) + A, ΔA = arrayify(A_ΔA) + trace = tr(A) + + function tr_pullback(Δtrace) + for (_, b) in blocks(ΔA) + TensorKit.diagview(b) .+= Δtrace + end + return NoRData(), NoRData() + end + + return CoDual(trace, Mooncake.NoFData()), tr_pullback +end diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index a1bce9906..e9f7d01d7 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -117,6 +117,14 @@ for V in spacelist Mooncake.TestUtils.test_rule(rng, norm, C, 2; atol, rtol, mode) Mooncake.TestUtils.test_rule(rng, norm, C', 2; atol, rtol, mode) + + D1 = randn(T, V[1] ← V[1]) + D2 = randn(T, V[1] ⊗ V[2] ← V[1] ⊗ V[2]) + D3 = randn(T, V[1] ⊗ V[2] ⊗ V[3] ← V[1] ⊗ V[2] ⊗ V[3]) + + Mooncake.TestUtils.test_rule(rng, tr, D1; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, tr, D2; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, tr, D3; atol, rtol, mode) end From d0e50b849d6fd48c833aa3597005ae8422f15bcb Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Thu, 22 Jan 2026 08:02:00 -0500 Subject: [PATCH 18/21] give up on planartrace for now --- ext/TensorKitMooncakeExt/planaroperations.jl | 34 +++++++++---- src/fusiontrees/manipulations.jl | 2 +- test/autodiff/mooncake.jl | 53 +++++++++++--------- 3 files changed, 52 insertions(+), 37 deletions(-) diff --git a/ext/TensorKitMooncakeExt/planaroperations.jl b/ext/TensorKitMooncakeExt/planaroperations.jl index a480293af..3d1742a3a 100644 --- a/ext/TensorKitMooncakeExt/planaroperations.jl +++ b/ext/TensorKitMooncakeExt/planaroperations.jl @@ -49,19 +49,31 @@ end planartrace_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) +# This implementation is slightly more involved than its non-planar counterpart +# this is because we lack a general `pAB` argument in `planarcontract`, and need +# to keep things planar along the way. +# In particular, we can't simply tensor product with multiple identities in one go +# if they aren't "contiguous", e.g. p = ((1, 4, 5), ()), q = ((2, 6), (3, 7)) function planartrace_pullback_ΔA!( ΔA, ΔC, A, p, q, α, backend, allocator ) - ip = invperm((linearize(p)..., q[1]..., q[2]...)) - pdA = _repartition(ip, A) - E = one!(TO.tensoralloc_add(scalartype(A), A, q, false)) - twist!(E, filter(x -> !isdual(space(E, x)), codomainind(E))) - pE = ((), trivtuple(TO.numind(q))) - pΔC = (trivtuple(TO.numind(p)), ()) - TensorKit.planarcontract!( - ΔA, ΔC, pΔC, E, pE, pdA, conj(α), One(), backend, allocator - ) - return NoRData() + if length(q[1]) == 0 + ip = invperm(linearize(p)) + pΔA = _repartition(ip, A) + TK.add_transpose!(ΔA, ΔC, pΔA, conj(α), One(), backend, allocator) + return NoRData() + end + # if length(q[1]) == 1 + # ip = invperm((p[1]..., q[2]..., p[2]..., q[1]...)) + # pdA = _repartition(ip, A) + # E = one!(TO.tensoralloc_add(scalartype(A), A, q, false)) + # twist!(E, filter(x -> !isdual(space(E, x)), codomainind(E))) + # # pE = ((), trivtuple(TO.numind(q))) + # # pΔC = (trivtuple(TO.numind(p)), ()) + # TensorKit.planaradd!(ΔA, ΔC ⊗ E, pdA, conj(α), One(), backend, allocator) + # return NoRData() + # end + error("The reverse rule for `planartrace` is not yet implemented") end function planartrace_pullback_Δα( @@ -73,7 +85,7 @@ function planartrace_pullback_Δα( # TODO: this result might be easier to compute as: # C′ = βC + α * trace(A) ⟹ At = (C′ - βC) / α At = TO.tensoralloc_add(scalartype(A), A, p, false, Val(true), allocator) - TensorKit.planartrace!(At, A, p, q, false, One(), backend, allocator) + TensorKit.planartrace!(At, A, p, q, One(), Zero(), backend, allocator) Δα = inner(At, ΔC) TO.tensorfree!(At, allocator) return Mooncake._rdata(Δα) diff --git a/src/fusiontrees/manipulations.jl b/src/fusiontrees/manipulations.jl index 1564b1b67..3cc6a16b6 100644 --- a/src/fusiontrees/manipulations.jl +++ b/src/fusiontrees/manipulations.jl @@ -692,7 +692,7 @@ function planar_trace( k += 1 end end - k > N₃ && throw(ArgumentError("Not a planar trace")) + k > N₃ && throw(ArgumentError(lazy"not a planar trace: ($q1, $q2)")) q1′ = let i = i, j = j map(l -> (l - (l > i) - (l > j)), TupleTools.deleteat(q1, k)) diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index e9f7d01d7..db7e0c078 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -325,30 +325,33 @@ for V in spacelist end end - @timedtestset "planartrace!" begin - for _ in 1:5 - k1 = rand(0:2) - k2 = rand(1:2) - V1 = map(v -> rand(Bool) ? v' : v, rand(V, k1)) - V2 = map(v -> rand(Bool) ? v' : v, rand(V, k2)) - - k′ = rand(0:(k1 + 2k2)) - (_p, _q) = randcircshift(k′, k1 + 2 * k2 - k′, k1) - p = _repartition(_p, rand(0:k1)) - q = _repartition(_q, k2) - ip = _repartition(invperm(linearize((_p, _q))), k′) - A = randn(T, permute(prod(V1) ⊗ prod(V2) ← prod(V2), ip)) - - α = randn(T) - β = randn(T) - C = randn!(TensorOperations.tensoralloc_add(T, A, p, false, Val(false))) - Mooncake.TestUtils.test_rule( - rng, TensorKit.planartrace!, - C, A, p, q, α, β, - TensorOperations.DefaultBackend(), TensorOperations.DefaultAllocator(); - atol, rtol, mode - ) - end - end + # TODO: currently broken + # @timedtestset "planartrace!" begin + # for _ in 1:5 + # k1 = rand(0:2) + # k2 = rand(0:1) + # V1 = map(v -> rand(Bool) ? v' : v, rand(V, k1)) + # V2 = map(v -> rand(Bool) ? v' : v, rand(V, k2)) + # V3 = prod(x -> x ⊗ x', V2[1:k2]; init = one(V[1])) + # V4 = prod(x -> x ⊗ x', V2[(k2 + 1):end]; init = one(V[1])) + # + # k′ = rand(0:(k1 + 2k2)) + # (_p, _q) = randcircshift(k′, k1 + 2k2 - k′, k1) + # p = _repartition(_p, rand(0:k1)) + # q = (tuple(_q[1:2:end]...), tuple(_q[2:2:end]...)) + # ip = _repartition(invperm(linearize((_p, _q))), k′) + # A = randn(T, permute(prod(V1) ⊗ V3 ← V4, ip)) + # + # α = randn(T) + # β = randn(T) + # C = randn!(TensorOperations.tensoralloc_add(T, A, p, false, Val(false))) + # Mooncake.TestUtils.test_rule( + # rng, TensorKit.planartrace!, + # C, A, p, q, α, β, + # TensorOperations.DefaultBackend(), TensorOperations.DefaultAllocator(); + # atol, rtol, mode + # ) + # end + # end end end From 2740a96e5446e51d35084b64ecdefec0ce16ce69 Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Thu, 22 Jan 2026 08:08:47 -0500 Subject: [PATCH 19/21] add rule `inv` --- ext/TensorKitMooncakeExt/linalg.jl | 15 +++++++++++++++ test/autodiff/mooncake.jl | 4 ++++ 2 files changed, 19 insertions(+) diff --git a/ext/TensorKitMooncakeExt/linalg.jl b/ext/TensorKitMooncakeExt/linalg.jl index 092ddf369..a35c1cea4 100644 --- a/ext/TensorKitMooncakeExt/linalg.jl +++ b/ext/TensorKitMooncakeExt/linalg.jl @@ -67,3 +67,18 @@ function Mooncake.rrule!!(::CoDual{typeof(tr)}, A_ΔA::CoDual{<:AbstractTensorMa return CoDual(trace, Mooncake.NoFData()), tr_pullback end + +Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(inv), AbstractTensorMap} + +function Mooncake.rrule!!(::CoDual{typeof(inv)}, A_ΔA::CoDual{<:AbstractTensorMap}) + A, ΔA = arrayify(A_ΔA) + Ainv_ΔAinv = Mooncake.zero_fcodual(inv(A)) + Ainv, ΔAinv = arrayify(Ainv_ΔAinv) + + function inv_pullback(::NoRData) + mul!(ΔA, Ainv' * ΔAinv, Ainv', -1, One()) + return NoRData(), NoRData() + end + + return Ainv_ΔAinv, inv_pullback +end diff --git a/test/autodiff/mooncake.jl b/test/autodiff/mooncake.jl index db7e0c078..0ae368235 100644 --- a/test/autodiff/mooncake.jl +++ b/test/autodiff/mooncake.jl @@ -125,6 +125,10 @@ for V in spacelist Mooncake.TestUtils.test_rule(rng, tr, D1; atol, rtol, mode) Mooncake.TestUtils.test_rule(rng, tr, D2; atol, rtol, mode) Mooncake.TestUtils.test_rule(rng, tr, D3; atol, rtol, mode) + + Mooncake.TestUtils.test_rule(rng, inv, D1; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, inv, D2; atol, rtol, mode) + Mooncake.TestUtils.test_rule(rng, inv, D3; atol, rtol, mode) end From d2f50a7d6d1e58932ff4616c9719bd958c34541b Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Thu, 22 Jan 2026 08:33:46 -0500 Subject: [PATCH 20/21] is_primitive in namespace --- .../TensorKitMooncakeExt.jl | 2 +- .../indexmanipulations.jl | 20 +++++++++---------- ext/TensorKitMooncakeExt/linalg.jl | 8 ++++---- ext/TensorKitMooncakeExt/planaroperations.jl | 2 +- ext/TensorKitMooncakeExt/tensoroperations.jl | 4 ++-- ext/TensorKitMooncakeExt/vectorinterface.jl | 8 ++++---- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl b/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl index 4c692adb9..d3894c874 100644 --- a/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl +++ b/ext/TensorKitMooncakeExt/TensorKitMooncakeExt.jl @@ -1,7 +1,7 @@ module TensorKitMooncakeExt using Mooncake -using Mooncake: @zero_derivative, DefaultCtx, ReverseMode, NoFData, NoRData, CoDual, arrayify, primal +using Mooncake: @zero_derivative, @is_primitive, DefaultCtx, ReverseMode, NoFData, NoRData, CoDual, arrayify, primal using TensorKit import TensorKit as TK using VectorInterface diff --git a/ext/TensorKitMooncakeExt/indexmanipulations.jl b/ext/TensorKitMooncakeExt/indexmanipulations.jl index 464c18392..39f7dd4fd 100644 --- a/ext/TensorKitMooncakeExt/indexmanipulations.jl +++ b/ext/TensorKitMooncakeExt/indexmanipulations.jl @@ -1,7 +1,7 @@ for transform in (:permute, :transpose) add_transform! = Symbol(:add_, transform, :!) add_transform_pullback = Symbol(add_transform!, :_pullback) - @eval Mooncake.@is_primitive( + @eval @is_primitive( DefaultCtx, ReverseMode, Tuple{ @@ -76,7 +76,7 @@ for transform in (:permute, :transpose) end end -Mooncake.@is_primitive( +@is_primitive( DefaultCtx, ReverseMode, Tuple{ @@ -153,8 +153,8 @@ function Mooncake.rrule!!( end # both are needed for correctly capturing every dispatch -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(twist!), AbstractTensorMap, Any} -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), @NamedTuple{inv::Bool}, typeof(twist!), AbstractTensorMap, Any} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(twist!), AbstractTensorMap, Any} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), @NamedTuple{inv::Bool}, typeof(twist!), AbstractTensorMap, Any} function Mooncake.rrule!!(::CoDual{typeof(twist!)}, t_Δt::CoDual{<:AbstractTensorMap}, inds_Δinds::CoDual) # prepare arguments @@ -198,8 +198,8 @@ function Mooncake.rrule!!( end # both are needed for correctly capturing every dispatch -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(flip), AbstractTensorMap, Any} -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), @NamedTuple{inv::Bool}, typeof(flip), AbstractTensorMap, Any} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(flip), AbstractTensorMap, Any} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), @NamedTuple{inv::Bool}, typeof(flip), AbstractTensorMap, Any} function Mooncake.rrule!!(::CoDual{typeof(flip)}, t_Δt::CoDual{<:AbstractTensorMap}, inds_Δinds::CoDual) # prepare arguments @@ -245,8 +245,8 @@ for insertunit in (:insertleftunit, :insertrightunit) insertunit_pullback = Symbol(insertunit, :_pullback) @eval begin # both are needed for correctly capturing every dispatch - Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof($insertunit), AbstractTensorMap, Val} - Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), NamedTuple, typeof($insertunit), AbstractTensorMap, Val} + @is_primitive DefaultCtx ReverseMode Tuple{typeof($insertunit), AbstractTensorMap, Val} + @is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), NamedTuple, typeof($insertunit), AbstractTensorMap, Val} function Mooncake.rrule!!(::CoDual{typeof($insertunit)}, tsrc_Δtsrc::CoDual{<:AbstractTensorMap}, ival_Δival::CoDual{<:Val}) # prepare arguments @@ -328,8 +328,8 @@ for insertunit in (:insertleftunit, :insertrightunit) end -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(removeunit), AbstractTensorMap, Val} -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), NamedTuple, typeof(removeunit), AbstractTensorMap, Val} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(removeunit), AbstractTensorMap, Val} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(Core.kwcall), NamedTuple, typeof(removeunit), AbstractTensorMap, Val} function Mooncake.rrule!!(::CoDual{typeof(removeunit)}, tsrc_Δtsrc::CoDual{<:AbstractTensorMap}, ival_Δival::CoDual{Val{i}}) where {i} # prepare arguments diff --git a/ext/TensorKitMooncakeExt/linalg.jl b/ext/TensorKitMooncakeExt/linalg.jl index a35c1cea4..a75e77922 100644 --- a/ext/TensorKitMooncakeExt/linalg.jl +++ b/ext/TensorKitMooncakeExt/linalg.jl @@ -1,4 +1,4 @@ -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(mul!), AbstractTensorMap, AbstractTensorMap, AbstractTensorMap, Number, Number} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(mul!), AbstractTensorMap, AbstractTensorMap, AbstractTensorMap, Number, Number} function Mooncake.rrule!!( ::CoDual{typeof(mul!)}, @@ -37,7 +37,7 @@ function Mooncake.rrule!!( return C_ΔC, mul_pullback end -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(norm), AbstractTensorMap, Real} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(norm), AbstractTensorMap, Real} function Mooncake.rrule!!(::CoDual{typeof(norm)}, tΔt::CoDual{<:AbstractTensorMap}, pdp::CoDual{<:Real}) t, Δt = arrayify(tΔt) @@ -52,7 +52,7 @@ function Mooncake.rrule!!(::CoDual{typeof(norm)}, tΔt::CoDual{<:AbstractTensorM return CoDual(n, Mooncake.NoFData()), norm_pullback end -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(tr), AbstractTensorMap} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(tr), AbstractTensorMap} function Mooncake.rrule!!(::CoDual{typeof(tr)}, A_ΔA::CoDual{<:AbstractTensorMap}) A, ΔA = arrayify(A_ΔA) @@ -68,7 +68,7 @@ function Mooncake.rrule!!(::CoDual{typeof(tr)}, A_ΔA::CoDual{<:AbstractTensorMa return CoDual(trace, Mooncake.NoFData()), tr_pullback end -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(inv), AbstractTensorMap} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(inv), AbstractTensorMap} function Mooncake.rrule!!(::CoDual{typeof(inv)}, A_ΔA::CoDual{<:AbstractTensorMap}) A, ΔA = arrayify(A_ΔA) diff --git a/ext/TensorKitMooncakeExt/planaroperations.jl b/ext/TensorKitMooncakeExt/planaroperations.jl index 3d1742a3a..df75d60fe 100644 --- a/ext/TensorKitMooncakeExt/planaroperations.jl +++ b/ext/TensorKitMooncakeExt/planaroperations.jl @@ -1,6 +1,6 @@ # planartrace! # ------------ -Mooncake.@is_primitive( +@is_primitive( DefaultCtx, ReverseMode, Tuple{ diff --git a/ext/TensorKitMooncakeExt/tensoroperations.jl b/ext/TensorKitMooncakeExt/tensoroperations.jl index 59a398e27..e38271200 100644 --- a/ext/TensorKitMooncakeExt/tensoroperations.jl +++ b/ext/TensorKitMooncakeExt/tensoroperations.jl @@ -1,6 +1,6 @@ # tensorcontract! # --------------- -Mooncake.@is_primitive( +@is_primitive( DefaultCtx, ReverseMode, Tuple{ @@ -135,7 +135,7 @@ end # tensortrace! # ------------ -Mooncake.@is_primitive( +@is_primitive( DefaultCtx, ReverseMode, Tuple{ diff --git a/ext/TensorKitMooncakeExt/vectorinterface.jl b/ext/TensorKitMooncakeExt/vectorinterface.jl index 2c1bfe984..625aadd61 100644 --- a/ext/TensorKitMooncakeExt/vectorinterface.jl +++ b/ext/TensorKitMooncakeExt/vectorinterface.jl @@ -1,4 +1,4 @@ -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(scale!), AbstractTensorMap, Number} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(scale!), AbstractTensorMap, Number} function Mooncake.rrule!!(::CoDual{typeof(scale!)}, C_ΔC::CoDual{<:AbstractTensorMap}, α_Δα::CoDual{<:Number}) # prepare arguments @@ -20,7 +20,7 @@ function Mooncake.rrule!!(::CoDual{typeof(scale!)}, C_ΔC::CoDual{<:AbstractTens return C_ΔC, scale_pullback end -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(scale!), AbstractTensorMap, AbstractTensorMap, Number} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(scale!), AbstractTensorMap, AbstractTensorMap, Number} function Mooncake.rrule!!(::CoDual{typeof(scale!)}, C_ΔC::CoDual{<:AbstractTensorMap}, A_ΔA::CoDual{<:AbstractTensorMap}, α_Δα::CoDual{<:Number}) # prepare arguments @@ -44,7 +44,7 @@ function Mooncake.rrule!!(::CoDual{typeof(scale!)}, C_ΔC::CoDual{<:AbstractTens return C_ΔC, scale_pullback end -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(add!), AbstractTensorMap, AbstractTensorMap, Number, Number} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(add!), AbstractTensorMap, AbstractTensorMap, Number, Number} function Mooncake.rrule!!(::CoDual{typeof(add!)}, C_ΔC::CoDual{<:AbstractTensorMap}, A_ΔA::CoDual{<:AbstractTensorMap}, α_Δα::CoDual{<:Number}, β_Δβ::CoDual{<:Number}) # prepare arguments @@ -73,7 +73,7 @@ function Mooncake.rrule!!(::CoDual{typeof(add!)}, C_ΔC::CoDual{<:AbstractTensor return C_ΔC, add_pullback end -Mooncake.@is_primitive DefaultCtx ReverseMode Tuple{typeof(inner), AbstractTensorMap, AbstractTensorMap} +@is_primitive DefaultCtx ReverseMode Tuple{typeof(inner), AbstractTensorMap, AbstractTensorMap} function Mooncake.rrule!!(::CoDual{typeof(inner)}, A_ΔA::CoDual{<:AbstractTensorMap}, B_ΔB::CoDual{<:AbstractTensorMap}) # prepare arguments From 0ec5584e27a64ec15619724dec439498f0ccac9b Mon Sep 17 00:00:00 2001 From: Lukas Devos Date: Thu, 22 Jan 2026 08:49:50 -0500 Subject: [PATCH 21/21] share more code --- .../indexmanipulations.jl | 44 +++++-------------- ext/TensorKitMooncakeExt/linalg.jl | 10 +++-- ext/TensorKitMooncakeExt/tensoroperations.jl | 28 ++---------- ext/TensorKitMooncakeExt/utility.jl | 6 +-- 4 files changed, 26 insertions(+), 62 deletions(-) diff --git a/ext/TensorKitMooncakeExt/indexmanipulations.jl b/ext/TensorKitMooncakeExt/indexmanipulations.jl index 39f7dd4fd..8a97ac81c 100644 --- a/ext/TensorKitMooncakeExt/indexmanipulations.jl +++ b/ext/TensorKitMooncakeExt/indexmanipulations.jl @@ -31,22 +31,18 @@ for transform in (:permute, :transpose) # if we need to compute Δa, it is faster to allocate an intermediate permuted A # and store that instead of repeating the permutation in the pullback each time. # effectively, we replace `add_permute` by `add ∘ permute`. - Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) - Ap = if Tdα === NoRData - TK.$add_transform!(C, A, p, α, β, ba...) - nothing - else + Ap = if _needs_tangent(α) Ap = $transform(A, p) add!(C, Ap, α, β) Ap + else + TK.$add_transform!(C, A, p, α, β, ba...) + nothing end function $add_transform_pullback(::NoRData) copy!(C, C_cache) - scale!(ΔC, conj(β)) - ΔCr = NoRData() - # ΔA ip = invperm(linearize(p)) pΔA = _repartition(ip, A) @@ -60,14 +56,8 @@ for transform in (:permute, :transpose) Mooncake._rdata(inner(Ap, ΔC)) end - # Δβ - Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) - Δβr = if Tdβ === NoRData - NoRData() - else - Mooncake._rdata(inner(C, ΔC)) - end - + Δβr = pullback_dβ(C, ΔC, β) + ΔCr = pullback_dC!(ΔC, β) return NoRData(), ΔCr, ΔAr, NoRData(), Δαr, Δβr, map(Returns(NoRData()), ba)... end @@ -107,22 +97,18 @@ function Mooncake.rrule!!( # if we need to compute Δa, it is faster to allocate an intermediate braided A # and store that instead of repeating the permutation in the pullback each time. # effectively, we replace `add_permute` by `add ∘ permute`. - Tdα = Mooncake.rdata_type(Mooncake.tangent_type(typeof(α))) - Ap = if Tdα === NoRData - TK.add_braid!(C, A, p, levels, α, β, ba...) - nothing - else + Ap = if _needs_tangent(α) Ap = braid(A, p, levels) add!(C, Ap, α, β) Ap + else + TK.add_braid!(C, A, p, levels, α, β, ba...) + nothing end function add_braid!_pullback(::NoRData) copy!(C, C_cache) - scale!(ΔC, conj(β)) - ΔCr = NoRData() - # ΔA ip = invperm(linearize(p)) pΔA = _repartition(ip, A) @@ -137,14 +123,8 @@ function Mooncake.rrule!!( Mooncake._rdata(inner(Ap, ΔC)) end - # Δβ - Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) - Δβr = if Tdβ === NoRData - NoRData() - else - Mooncake._rdata(inner(C, ΔC)) - end - + Δβr = pullback_dβ(C, ΔC, β) + ΔCr = pullback_dC!(ΔC, β) return NoRData(), ΔCr, ΔAr, NoRData(), NoRData(), Δαr, Δβr, map(Returns(NoRData()), ba)... end diff --git a/ext/TensorKitMooncakeExt/linalg.jl b/ext/TensorKitMooncakeExt/linalg.jl index a75e77922..2a77792c9 100644 --- a/ext/TensorKitMooncakeExt/linalg.jl +++ b/ext/TensorKitMooncakeExt/linalg.jl @@ -1,3 +1,8 @@ +# Shared +# ------ +pullback_dC!(ΔC, β) = (scale!(ΔC, conj(β)); return NoRData()) +pullback_dβ(C, ΔC, β) = _needs_tangent(β) ? inner(C, ΔC) : NoRData() + @is_primitive DefaultCtx ReverseMode Tuple{typeof(mul!), AbstractTensorMap, AbstractTensorMap, AbstractTensorMap, Number, Number} function Mooncake.rrule!!( @@ -22,14 +27,13 @@ function Mooncake.rrule!!( function mul_pullback(::NoRData) copy!(C, C_cache) - scale!(ΔC, conj(β)) mul!(ΔA, ΔC, B', conj(α), One()) mul!(ΔB, A', ΔC, conj(α), One()) - ΔCr = NoRData() ΔAr = NoRData() ΔBr = NoRData() Δαr = isnothing(AB) ? NoRData() : Mooncake._rdata(inner(AB, ΔC)) - Δβr = _needs_tangent(β) ? Mooncake._rdata(inner(C, ΔC)) : NoRData() + Δβr = pullback_dβ(C, ΔC, β) + ΔCr = pullback_dC!(ΔC, β) return NoRData(), ΔCr, ΔAr, ΔBr, Δαr, Δβr end diff --git a/ext/TensorKitMooncakeExt/tensoroperations.jl b/ext/TensorKitMooncakeExt/tensoroperations.jl index e38271200..66c3f257a 100644 --- a/ext/TensorKitMooncakeExt/tensoroperations.jl +++ b/ext/TensorKitMooncakeExt/tensoroperations.jl @@ -45,8 +45,8 @@ function Mooncake.rrule!!( Δαr = blas_contract_pullback_Δα( ΔC, A, pA, B, pB, pAB, α, backend, allocator ) - Δβr = blas_contract_pullback_Δβ(ΔC, C, β) - ΔCr = blas_contract_pullback_ΔC!(ΔC, β) + Δβr = pullback_dβ(ΔC, C, β) + ΔCr = pullback_dC!(ΔC, β) return NoRData(), ΔCr, ΔAr, NoRData(), @@ -59,8 +59,6 @@ function Mooncake.rrule!!( return C_ΔC, blas_contract_pullback end -blas_contract_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) - function blas_contract_pullback_ΔA!( ΔA, ΔC, A, pA, B, pB, pAB, α, backend, allocator ) @@ -125,14 +123,6 @@ function blas_contract_pullback_Δα( return Mooncake._rdata(Δα) end -function blas_contract_pullback_Δβ(ΔC, C, β) - Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) - Tdβ === NoRData && return NoRData() - - Δβ = inner(C, ΔC) - return Mooncake._rdata(Δβ) -end - # tensortrace! # ------------ @is_primitive( @@ -171,8 +161,8 @@ function Mooncake.rrule!!( ΔAr = trace_permute_pullback_ΔA!(ΔA, ΔC, A, p, q, α, backend) Δαr = trace_permute_pullback_Δα(ΔC, A, p, q, α, backend) - Δβr = trace_permute_pullback_Δβ(ΔC, C, β) - ΔCr = trace_permute_pullback_ΔC!(ΔC, β) + Δβr = pullback_dβ(ΔC, C, β) + ΔCr = pullback_dC!(ΔC, β) return NoRData(), ΔCr, ΔAr, NoRData(), NoRData(), @@ -182,8 +172,6 @@ function Mooncake.rrule!!( return C_ΔC, trace_permute_pullback end -trace_permute_pullback_ΔC!(ΔC, β) = (scale!(ΔC, conj(β)); NoRData()) - function trace_permute_pullback_ΔA!( ΔA, ΔC, A, p, q, α, backend ) @@ -211,11 +199,3 @@ function trace_permute_pullback_Δα( Δα = inner(At, ΔC) return Mooncake._rdata(Δα) end - -function trace_permute_pullback_Δβ(ΔC, C, β) - Tdβ = Mooncake.rdata_type(Mooncake.tangent_type(typeof(β))) - Tdβ === NoRData && return NoRData() - - Δβ = inner(C, ΔC) - return Mooncake._rdata(Δβ) -end diff --git a/ext/TensorKitMooncakeExt/utility.jl b/ext/TensorKitMooncakeExt/utility.jl index e93de22be..261c1dcc2 100644 --- a/ext/TensorKitMooncakeExt/utility.jl +++ b/ext/TensorKitMooncakeExt/utility.jl @@ -1,7 +1,7 @@ _needs_tangent(x) = _needs_tangent(typeof(x)) -_needs_tangent(::Type{<:Number}) = true -_needs_tangent(::Type{<:Integer}) = false -_needs_tangent(::Type{<:Union{One, Zero}}) = false +function _needs_tangent(::Type{T}) where {T <: Number} + return Mooncake.rdata_type(Mooncake.tangent_type(T)) !== NoRData() +end # IndexTuple utility # ------------------