What is the size required for ti.dot?

综合讨论区
#1

I am wondering what is the input size required for ti.dot?
In addition, what is the size that required to assign an entry of a matrix? For example, if I have a tensor a (1,1) and a tensor b(6, 1), can I simply use b[0] = a?

The following is a two layer network with some bugs. Since it is suggested not to use large matrices, I am trying to decompose the matrix multiplication to dot production of vectors.

import taichi as ti

real = ti.f32
ti.set_default_fp(real)
ti.get_runtime().set_verbose(True)

in_size = 2
hidden_size = 4

ti_dummy_data = ti.Vector(in_size, dt=real, shape=(), needs_grad=True)
ti_weight_0 = ti.Vector(in_size, dt=real, shape=(hidden_size), needs_grad=True)
ti_bias_0 = ti.Vector(hidden_size, dt=real, shape=(), needs_grad=True)
ti_out_0 = ti.Vector(hidden_size, dt=real, shape=(), needs_grad=True)
ti_weight_1 = ti.Vector(hidden_size, dt=real, shape=(), needs_grad=True)
ti_bias_1 = ti.Vector(1, dt=real, shape=(), needs_grad=True)
ti_out_1 = ti.Vector(1, dt=real, shape=(), needs_grad=True)

@ti.kernel
def net():
    for i in range(hidden_size):  # 4
        ti_out_0[i] = ti.max([ti.dot(ti_dummy_data[None], ti_weight_0[i])] + ti_bias_0[None][i], 0)
    ti_out_1[0] = ti.dot(ti_out_0[None], ti_weight_1[None]) + ti_bias_1[None]
net()

It gives the errors:

[Release mode]
[Taichi version 0.3.2, cpu only, commit 26635f82]
Initializing runtime with 44 elements
Runtime initialized.
Traceback (most recent call last):
  File "D:/learn_mpm/test.py", line 23, in <module>
    net()
  File "C:\Users\49446\AppData\Local\conda\conda\envs\py36\lib\site-packages\taichi\lang\kernel.py", line 308, in __call__
    self.materialize(key=key, args=args, arg_features=self.mapper.extract(args))
  File "C:\Users\49446\AppData\Local\conda\conda\envs\py36\lib\site-packages\taichi\lang\kernel.py", line 233, in materialize
    taichi_kernel = taichi_kernel.define(taichi_ast_generator)
  File "C:\Users\49446\AppData\Local\conda\conda\envs\py36\lib\site-packages\taichi\lang\kernel.py", line 230, in taichi_ast_generator
    compiled()
  File "D:/learn_mpm/test.py", line 21, in net
    ti_out_0[i] = ti.max([ti.dot(ti_dummy_data[None], ti_weight_0[i])] + ti_bias_0[None][i], 0)
  File "C:\Users\49446\AppData\Local\conda\conda\envs\py36\lib\site-packages\taichi\lang\impl.py", line 52, in subscript
    return value.subscript(*indices)
  File "C:\Users\49446\AppData\Local\conda\conda\envs\py36\lib\site-packages\taichi\lang\matrix.py", line 238, in subscript
    ret.entries[i] = impl.subscript(e, *indices)
  File "C:\Users\49446\AppData\Local\conda\conda\envs\py36\lib\site-packages\taichi\lang\impl.py", line 58, in subscript
    assert int(value.ptr.get_attribute("dim")) == indices_expr_group.size()
AssertionError

Process finished with exit code 1

I also wrote this in ti.var, but it broke down with some strange errors.

import taichi as ti

real = ti.f32
ti.set_default_fp(real)
ti.get_runtime().set_verbose(True)

in_size = 2
hidden_size = 4

ti_dummy_data = ti.var(dt=real, shape=in_size, needs_grad=True)
ti_weight_0 = ti.var(dt=real, shape=(hidden_size, in_size), needs_grad=True)
ti_bias_0 = ti.var(dt=real, shape=hidden_size, needs_grad=True)
ti_out_0 = ti.var(dt=real, shape=hidden_size, needs_grad=True)
ti_weight_1 = ti.var(dt=real, shape=hidden_size, needs_grad=True)
ti_bias_1 = ti.var(dt=real, shape=1, needs_grad=True)
ti_out_1 = ti.var(dt=real, shape=1, needs_grad=True)


@ti.kernel
def net():
    for i in range(hidden_size):  # 4
        dummy = ti.dot(ti.Vector(ti_dummy_data), ti.Vector(ti_weight_0[i]))
        dummy += ti_bias_0[i]
        dummy = ti.max(dummy, 0)
        ti_out_0[i] = dummy
    ti_out_1[None] = ti.dot(ti.Vector(ti_out_0), ti.Vector(ti_weight_1)) + ti_bias_1
net()

Errors:

[Taichi version 0.3.2, cpu only, commit 26635f82]
IMAGE_REL_AMD64_ADDR32NB relocation requires anordered section layout.
Initializing runtime with 29 elements
Runtime initialized.




Process finished with exit code -1
#2

Q: I am wondering what is the input size required for ti.dot ?
A: ti.dot works for two matrices of the same size (nx1)

Q: In addition, what is the size that required to assign an entry of a matrix? For example, if I have a tensor a (1,1) and a tensor b(6, 1), can I simply use b[0] = a ?"
A: No, matrices and scalars are completely difference concepts. Please use b[0] = a[0, 0].

#3

The right way to write the first script:

import taichi as ti

real = ti.f32
ti.set_default_fp(real)
ti.get_runtime().set_verbose(True)

in_size = 2
hidden_size = 4

ti_dummy_data = ti.Vector(in_size, dt=real, shape=(), needs_grad=True)
ti_weight_0 = ti.Vector(in_size, dt=real, shape=(hidden_size), needs_grad=True)
ti_bias_0 = ti.Vector(1, dt=real, shape=(hidden_size), needs_grad=True)
ti_out_0 = ti.Vector(hidden_size, dt=real, shape=(), needs_grad=True)
ti_weight_1 = ti.Vector(hidden_size, dt=real, shape=(), needs_grad=True)
ti_bias_1 = ti.Vector(1, dt=real, shape=(), needs_grad=True)
ti_out_1 = ti.Vector(1, dt=real, shape=(), needs_grad=True)

@ti.kernel
def net():
  for i in ti.static(range(hidden_size)):  # 4
    ti_out_0[None][i] = ti.max(ti.dot(ti_dummy_data[None], ti_weight_0[i]) + ti_bias_0[i][0], 0)
  ti_out_1[None][0] = ti.dot(ti_out_0[None], ti_weight_1[None]) + ti_bias_1[None][0]

net()
#4

The right way to write the second script:

import taichi as ti

real = ti.f32
ti.set_default_fp(real)
ti.get_runtime().set_verbose(True)

in_size = 2
hidden_size = 4

ti_dummy_data = ti.var(dt=real, shape=in_size, needs_grad=True)
ti_weight_0 = ti.var(dt=real, shape=(hidden_size, in_size), needs_grad=True)
ti_bias_0 = ti.var(dt=real, shape=hidden_size, needs_grad=True)
ti_out_0 = ti.var(dt=real, shape=hidden_size, needs_grad=True)
ti_weight_1 = ti.var(dt=real, shape=(1, hidden_size), needs_grad=True)
ti_bias_1 = ti.var(dt=real, shape=1, needs_grad=True)
ti_out_1 = ti.var(dt=real, shape=1, needs_grad=True)

@ti.kernel
def net():
    for i in range(hidden_size):  # 4
        s = 0.0
        for j in ti.static(range(in_size)): # use ti.static for autodiff
            s += ti_dummy_data[j] * ti_weight_0[i, j]
        s += ti_bias_0[i]
        ti_out_0[i] = ti.max(s, 0)

    for i in range(1):
        s = 0.0
        for j in ti.static(range(hidden_size)):
            s += ti_out_0[j] * ti_weight_1[i, j]
        s += ti_bias_1[i]
        ti_out_1[i] = s

net()
#5

In the first script, what is the function of None in ti_out_0[None][i]?

#6

Please check out https://taichi.readthedocs.io/en/latest/tensor_matrix.html#tensors-of-scalars

#7

I seem to understand it. Do I have to specify the index of shape, even if shape=() in which case the index is None? The ‘0’ in the phrase ‘0-D tensor’ is corresponding to shape=()?

#8

Yes, you got it. 0-D tensors have empty “()” shapes.

#9

Thank you so much!!!