Tensors

1. What is Tensor

A tensor is a mathematical object that generalizes scalars, vectors, and matrices to higher dimensions. It’s essentially a container for data that can be indexed in multiple dimensions.

scalar-vector-matrix-tensor.png

Tensor vs ndarray

  • Tensor is similar to the fundamental object in Numpy called ndarray

  • ndarray is defined as an n-dimensional homogeneous array of fixed-size items

Advantages of Tensor

  • Tensor operations are performed significantly faster using GPUs

  • Tensors can be stored and manipulated at scale using distributed processing on multi GPUs and GPUs and across multiple servers

  • Tensors keep track of the graph of computations that created them

  • Tensors are much more than a special sort of multi-dimensional arrays

  • Interactions with each other such that transforming the tensors as a whole means that each tensor follows a particular transformation rule

2. Creat Tensors

2.1 From Python objects

import torch
print(torch.__version__)

zero_tensor = torch.tensor([2])
oneD_tensor = torch.tensor((1,2,3))
twoD_tensor = torch.tensor([[1,2,3], [4,5,6]]) # can be list or tuple
treD_tensor = torch.tensor([[[1,2,3], [4,5,6]], [[7,8,9], [10,11,12]]])

# tensor.size()
print(f'zero_tensor = {zero_tensor} and its tensor size = {zero_tensor.size()}\n')
print(f'oneD_tensor = {oneD_tensor} and its tensor size = {oneD_tensor.size()}\n')
print(f'twoD_tensor = {twoD_tensor} and its tensor size = {twoD_tensor.size()}\n')
print(f'treD_tensor = {treD_tensor} and its tensor size = {treD_tensor.size()}')

2.2 From Numpy objects

# pytorch

import numpy as np

twoD_tensor = torch.tensor(np.array([1,3,5,7,9]))
print(f'twoD_tensor = {twoD_tensor} and its tensor size = {twoD_tensor.size()}\n')

twoD_tensor = torch.tensor(np.arange(1, 10, 2))
print(f'twoD_tensor = {twoD_tensor} and its tensor size = {twoD_tensor.size()}\n')

np_array = np.array([1, 2, 3, 4, 5])
tensor_x = torch.from_numpy(np_array)
print(f'tensor_x = {tensor_x} and its tensor size = {tensor_x.size()}\n')

# tensorflow

import tensorflow as tf

# from a Python list
tensor_tf1 = tf.constant([1, 2, 3, 4])
print(tensor_tf1)

# from a NumPy array
tensor_tf2 = tf.convert_to_tensor(np.array([[1, 2], [3, 4]]), dtype=tf.float32)
print(tensor_tf2)

2.3 From functions to create tensors

  • torch.zeros() and torch.zeros_like(), tf.zeros() and tf.zeros_like()

  • torch.ones() and torch.ones_like(), tf.ones() and tf.ones_like()

  • torch.empty() and torch.rand_like()

  • torch.full() and torch.full_like()

empty_tensor = torch.empty(3,3)
print(f'empty_tensor = {empty_tensor} and its tensor size = {empty_tensor.size()}\n')

zeros_tensor = torch.zeros(3,3)
print(f'zeros_tensor = {zeros_tensor} and its tensor size = {zeros_tensor.size()}\n')

ones_tensor = torch.ones(3,3)
print(f'ones_tensor = {ones_tensor} and its tensor size = {ones_tensor.size()}\n')

full_tensor = torch.full((3,3), 0.12)
print(f'full_tensor = {full_tensor} and its tensor size = {full_tensor.size()}\n')

print('----------------------------------------------------\n')

a_tensor = torch.rand(2, 4)
print(f'a_tensor = {a_tensor} and its tensor size = {a_tensor.size()}\n')

a_zeros = torch.zeros_like(a_tensor)
print(f'a_zeros = {a_zeros} and its tensor size = {a_zeros.size()}\n')

a_ones = torch.ones_like(a_tensor)
print(f'a_ones = {a_ones} and its tensor size = {a_ones.size()}\n')

a_rand = torch.rand_like(a_tensor)
print(f'a_rand = {a_rand} and its tensor size = {a_rand.size()}\n')

a_full = torch.full_like(a_rand, 7) # here this matrix should be filled with `7`
print(f'a_full = {a_full} and its tensor size = {a_full.size()}\n')

# generating tensor using tensorflow

import tensorflow as tf

zeros_tensor = tf.zeros([3,3])
print(f'zeros_tensor = {zeros_tensor} and its tensor shape = {zeros_tensor.shape}\n')

ones_tensor = tf.ones([3,3])
print(f'ones_tensor = {ones_tensor} and its tensor shape = {ones_tensor.shape}\n')

# torch.eye(n)

tensor_eye = torch.eye(3)
print(f'tensor_eyee = \n{tensor_eye}\n and its tensor size = {tensor_eye.size()}\n')

# torch.linspace(start, end, steps)
# torch.logspace(start, end, steps)

lin = torch.linspace(0, 23, steps=5)
print(f'lin = {lin} and its tensor size = {lin.size()}\n')

log = torch.logspace(-3, 7, steps=4)
print(f'log = {log} and its tensor size = {log.size()}\n')

2.4 Random tensors

torch.manual_seed(123456) # to keep data reproduciable

# generate values from a uniform distribution on the interval [0, 1)
a_tensor = torch.rand(3,3)
print(f'a_tensor = {a_tensor} and its tensor size = {a_tensor.size()}\n')

# standard normal distribution
a_tensor = torch.randn(3,3)
print(f'a_tensor = {a_tensor} and its tensor size = {a_tensor.size()}\n')

# `torch.randint(a, b, (m, n))` generate a `m*n` integers from `a` to `b`
a_tensor = torch.randint(1, 10, (4,5))
print(f'a_tensor = {a_tensor} and its tensor size = {a_tensor.size()}\n')

# `torch.randperm(n)` to create a tensor with a random permutation of integers
a_tensor = torch.randperm(10)
print(f'a_tensor = {a_tensor} and its tensor size = {a_tensor.size()}\n')

3. Tensor’s properties

3.1 Tensor.shape

a_tensor = torch.rand(3,3,3)
print(f'a_tensor = {a_tensor} and its tensor shape = {a_tensor.shape}\n')

a_tensor = torch.rand(3, 3)
print(f'a_tensor = {a_tensor} and its tensor shape = {a_tensor.shape}\n')

a_tensor = torch.rand(3)
print(f'a_tensor = {a_tensor} and its tensor shape = {a_tensor.shape}\n')

a_tensor = torch.rand(1)
print(f'a_tensor = {a_tensor} and its tensor shape = {a_tensor.shape}')

3.2 Tensor.ndim

a_tensor = torch.rand(3,3,3)
print(f'a_tensor = {a_tensor} and its tensor ndim = {a_tensor.ndim}\n')

a_tensor = torch.rand(3, 3)
print(f'a_tensor = {a_tensor} and its tensor ndim = {a_tensor.ndim}\n')

a_tensor = torch.rand(3)
print(f'a_tensor = {a_tensor} and its tensor ndim = {a_tensor.ndim}\n')

a_tensor = torch.rand(1)
print(f'a_tensor = {a_tensor} and its tensor ndim = {a_tensor.ndim}')

3.3 Tensor.dtype

a_tensor_int = torch.tensor([1,2,3,4,5])
print(f'a_tensor_int = {a_tensor_int} and its tensor dtype = {a_tensor_int.dtype}\n')

b_tensor_flt = torch.tensor([1,2,3,4,5], dtype=torch.float) # can use `float64`
print(f'b_tensor_flt = {b_tensor_flt} and its tensor dtype = {b_tensor_flt.dtype}\n')

# torch.int8, torch.int16, torch.int32, torch.int64, torch.float16, torch.float32, torch.float64

print('-----------------------------------------------------------------\n')

# data type conversion
print('------- data type conversion (two methods) -------\n')

a_tensor_flt1 = a_tensor_int.float()
print(f'a_tensor_flt1 = {a_tensor_flt1} and its tensor dtype = {a_tensor_flt1.dtype}\n')

a_tensor_flt1 = a_tensor_int.to(torch.float64) # or .to(dtype=torch.double)
print(f'a_tensor_flt1 = {a_tensor_flt1} and its tensor dtype = {a_tensor_flt1.dtype}\n')

4. Tensor operations

4.1 Indexing

torch.manual_seed(123456)

a_tensor = torch.rand(5,5)
print(f'a_tensor = {a_tensor} and its tensor shape = {a_tensor.shape}\n')

a_tensor_row = a_tensor[2]
print(f"A ROW of a_tensor = {a_tensor_row}\n")

a_tensor_col = a_tensor[:, 2]
print(f"A COL of a_tensor = {a_tensor_col}\n")

a_tensor_cubic = a_tensor[1:4, 1:4]
print(f"A inner cubic of a_tensor = \n{a_tensor_cubic}\n")

# use indexing  to extract data that meets some criteria
print("Extra data > 0.5 = ", a_tensor_cubic > 0.5)
print("Extra data > 0.5 = ", a_tensor_cubic[a_tensor_cubic > 0.5])

4.2 Combining Tensors

a_tensor = torch.tensor([[0,1,2],[3,4,5]])
print(f'a_tensor = {a_tensor}\n')

b_tensor = torch.tensor([[10,11,12],[13,14,15]])
print(f'b_tensor = {b_tensor}\n')

# using `torch.stack` to add new dimension and stacks -- creates a new dimension
stack_tensor_dim0 = torch.stack((a_tensor, b_tensor), dim=0) # stack by rows
print(f'stack_tensor_dim0 = \n{stack_tensor_dim0} and its tensor shape = {stack_tensor_dim0.shape}\n')

stack_tensor_dim1 = torch.stack((a_tensor, b_tensor), dim=1) # stack by columns
print(f'stack_tensor_dim1 = \n{stack_tensor_dim1} and its tensor shape = {stack_tensor_dim1.shape}\n')

# using `torch.cat` to concatenates along existing dim -- extends an existing dimension.
cat_tensor_dim0 = torch.cat((a_tensor, b_tensor), dim=0) # cat by rows
print(f'cat_tensor_dim0 = \n{cat_tensor_dim0} and its tensor shape = {cat_tensor_dim0.shape}\n')

cat_tensor_dim1 = torch.cat((a_tensor, b_tensor), dim=1) # cat by columns
print(f'cat_tensor_dim1 = \n{cat_tensor_dim1} and its tensor shape = {cat_tensor_dim1.shape}\n')

4.3 Split Tensors

Function

Purpose

Input Parameters

Output

Changes Shape?

Splits Tensor?

torch.chunk

split into fixed number of chunks

chunks (int), dim

Tuple of tensors

No

Yes

torch.split

split into chunks of specified sizes

split_size_or_sections, dim

Tuple of tensors

No

Yes

torch.view

reshape tensor

New shape

Single tensor

Yes

No

torch.unbind

remove a dimension, split into slices

dim

Tuple of tensors

Yes (removes dim)

Yes

 
torch.manual_seed(101)

a_tensor = torch.rand(5,5)
print(f"a_tensor = {a_tensor}\n")

print('--------- torch.unbind(tensor) ---------')
tensors = torch.unbind(a_tensor, dim=1)
print(f"first_tensor = {tensors}")

tensors = torch.unbind(a_tensor, dim=0)
print(f"first_tensor = {tensors}")

print('--------- torch.split(tensor) ---------')
tensors = torch.split(a_tensor, 3, dim=0) # split by rows
print(f"tensors = {tensors}\n")

tensors = torch.split(a_tensor, 3, dim=1) # split by cols
print(f"tensors = {tensors}\n")

print('--------- torch.chunk(tensor) ---------')

chunks = torch.chunk(a_tensor, 3, dim=0)
print(f"chunks = {chunks}\n")
print(f"chunks[0] = {chunks[0]}")
print(f"chunks[1] = {chunks[1]}")
print(f"chunks[2] = {chunks[2]}\n")

chunks = torch.chunk(a_tensor, 3, dim=1)
print(f"chunks = {chunks}\n")

# `torch.view` provides an easy way to reshape a tensor

a_tensor = torch.randn(9)
print(f"a_tensor = \n{a_tensor}\n")
b_tensor = a_tensor.view(3,3)
print(f"b_tensor = \n{b_tensor}\n")

# `torch.flatten` method can be used to collapse the dimensions of a given tensor starting with a particular dimension
a_tensor = torch.randn(2,3,4)
print(f"a_tensor = \n{a_tensor}\n")
b_tensor = torch.flatten(a_tensor, start_dim=0)
print(f"b_tensor = \n{b_tensor}\n")
c_tensor = torch.flatten(a_tensor, start_dim=1)
print(f"c_tensor = \n{c_tensor}\n")
d_tensor = torch.flatten(a_tensor, start_dim=2)
print(f"d_tensor = \n{d_tensor}\n")

print(f"tensor shapes = b:{b_tensor.shape}")
print(f"tensor shapes = c:{c_tensor.shape}")
print(f"tensor shapes = d:{d_tensor.shape}")

4.4 Build-in Math Functions

  • pointwise operations

  • reduction functions

  • comparison functions

  • linear algebra operations

  • spectral and other math computations

4.4.1 Pointwise operations

To perform an operation on each point in tensor individually and return a new tensor

  • basic math functions, like add(), sub(), mul(), div(), neg(), and true_divid()

  • functions for truncation, like ceil(), clamp(), floor(), etc.

  • logical functions

  • trigonometry functions, like sin(), cos(), etc.

a = torch.tensor((11,12,13,14,15))
b = torch.tensor([1,2,3,4,5])

print(f"tensor a = {a}")
print(f"tensor b = {b}\n")
print(f"tensor a+b = {a+b} and {a.add(b)}")
print(f"tensor a+b = {a-b} and {a.sub(b)}")
print(f"tensor a+b = {a*b} and {a.mul(b)}")
print(f"tensor a+b = {a/b} and {a.div(b)}\n")

print(f"Sin of tensor a = {torch.sin(a)}")
print(f"Cos of tensor b = {torch.cos(b)}")

a = torch.tensor((11,12,13))
b = torch.tensor([1,2,3])

# `dot()` allows you to compute the dot product of tensors
print(f"tensor dot = {torch.dot(a, b)}")

# `cross()` allows to compute cross product of two tensors
print(f"tensor cross = {torch.cross(a, b)}")

4.4.2 Reduction functions

Reduce numbers down to a single number or a smaller set of numbers

  • results in reducing the dimensionality or rank of a tensor

  • include statistical functions such as mean, median, mode, etc.

a = torch.tensor((11,12,13,14,15), dtype=torch.float)
b = torch.tensor([1,2,3,4,5])

print(f"Mean   of tensor a = {torch.mean(a)}")
print(f"Median of tensor b = {torch.median(b)}")
print(f"Mode   of tensor b = {torch.mode(b)}")
print(f"Standard deviation of tensor a = {torch.std(a)}")

4.4.3 Comparison functions

  • compare all values within a tensor or compare values of two different tensors

  • functions to find minimum or maximum value, sort tensor values, test tensor status or conditions, etc.

torch.manual_seed(123456)

a_tensor = torch.rand(2,3)
b_tensor = torch.rand(2,3)
print(f"a_tensor = \n{a_tensor}\n")
print(f"b_tensor = \n{b_tensor}\n")

ge_torch = torch.ge(a_tensor, b_tensor).long()
print(f"ge_torch = \n{ge_torch}\n")

lt_torch = torch.lt(a_tensor, b_tensor).long()
print(f"lt_torch = \n{lt_torch}\n")

# torch.le(), torch.eq(), torch.ne()

# `any` and `all` methods enable to check whether a given condition is true in any or all cases

a_tensor = torch.randn(3,3)
print(f"Tensor A = {a_tensor}\n")

print(torch.any(a > 0.0).long())
print(torch.any(a > 1.0))
print(torch.all(a > 0.0))
print(torch.all(a > 1.0))

4.4.4 Linear algebra operations

  • enable matrix operations and are essential for DL computations

  • functions for matrix computations and tensor computations

  • PyTorch has a module called torch.linalg than contains a set of build-in functions that are based on BLAS and LAPACK standardized libraries

# compute dot product (scalar) of two 1D tensors
a_tensor = torch.tensor([1, 2, 3, 4], dtype=torch.int32)
b_tensor = torch.tensor(np.arange(1,9,2), dtype=torch.int32)

ab_tensor = torch.matmul(a_tensor, b_tensor)
print(f"Torch.matmul for two 1D scalarss = {ab_tensor}")

# compute matrix-matrix product (2D tensor) of two 2D tensors
a_tensor = torch.tensor([[1, 2, 3], [4, 5, 6]])
b_tensor = torch.tensor([[1, 2], [3, 4], [5, 6]])

ab_tensor = torch.matmul(a_tensor, b_tensor)
print(f"Torch.matmul for two 2D matrices = {ab_tensor}\n")

# compute a matrix product of multiple 2D tensors
a_tensor = torch.rand(2, 3)
b_tensor = torch.rand(3, 4)
c_tensor = torch.rand(4, 5)
d_tensor = torch.rand(5, 6)
abcd_tensor = torch.matmul(torch.matmul(a_tensor, b_tensor), torch.matmul(c_tensor, d_tensor))
print(f"Torch.matmul for four 2D matrices = \n{abcd_tensor}")

abcd_tensor = torch.linalg.multi_dot((a_tensor, b_tensor, c_tensor, d_tensor))
print(f"Torch.linalg.multi_dot for four 2D matrices = \n{abcd_tensor}")

# compute eigenvalues and eigenvectors

a_tensor = torch.rand(4,4)

print(f"Tensor A = {a_tensor}\n")

eigenvalues, eigenvectors = torch.linalg.eigh(a_tensor)
print(f"Eigenvalues = {eigenvalues}\n")
print(f"Eigenvectors = {eigenvectors}")

# `addbmm` (bmm = batch matrix-matrix product) to perform computation p*m+q*[a1*b1+a2*b2+...]
# where p and q are scalars, and m, a1, b1, a2, and b2 are tensors

# Note that `addbmm` takes parameters p and q with default values equal to one
# and that tensors such as a1 and a2 are provided by stacking them along 1st dimension

a = torch.ones(2, 2, 3)
b = torch.ones(2, 3, 2)
m = torch.ones(2,2)
print(f"shapes of a, b, and m tensors are {a.shape}, {b.shape}, and {m.shape}\n")

print(torch.addbmm(2, m, 3, a, b), '\n')
print(torch.addbmm(1, m, 1, a, b))

# `baddbmm` to perform p1 * m + q * [a1 * b1], p2 * m + q * [a2 * b2], ...
# where p and q are scalars, and m, p1, a1, b1, p2, a2, and b2 are tensors

# Note that `baddbmm` takes parameters p and q with default values equal to one
# and that tensors such as p1, a1, and a2 are provided by stacking them along 1st dimension

a = torch.ones(2, 2, 3)
b = torch.ones(2, 3, 2)
m = torch.ones(2, 2, 2)
print(f"shapes of a, b, and m tensors are {a.shape}, {b.shape}, and {m.shape}\n")

print(torch.baddbmm(1, m, 1, a, b), '\n')
print(torch.baddbmm(2, m, 1, a, b), '\n')
print(torch.baddbmm(1, m, 2, a, b))

# `bmm` to perform batch-wise matrix multiplication for tensor

a = torch.ones(2, 2, 3)
b = torch.ones(2, 3, 2)
print(f"shapes of a, b tensors are {a.shape},and {b.shape}\n")

print(torch.bmm(a, b))

# `addmm` is a non-batch version of addbmm that allows to perform computation p * m + q * a * b
# where p and q are scalars, and m, a, and b are tensors

# Note that `addmm` takes parameters p and q with default values equal to one

a = torch.ones(2, 3)
b = torch.ones(3, 2)
m = torch.ones(2, 2)
print(f"shapes of a, b, and m tensors are {a.shape}, {b.shape}, and {m.shape}\n")

print(torch.addmm(m, a, b), '\n')
print(torch.addmm(2, m, 3, a, b), '\n')
print(torch.addmm(1, m, 1, a, b))

# `addmv` (matrix-vector) allows to perform computation p * m + q * a * b
# where p and q are scalars, m and a are matrices, and b is a vector

# Note that `addmv` takes parameters p and q with default values equal to one

a = torch.ones(2, 3)
b = torch.ones(3)
m = torch.ones(2)
print(f"shapes of a, b, and m tensors are {a.shape}, {b.shape}, and {m.shape}\n")

print(torch.addmv(m, a, b), '\n')
print(torch.addmv(2, m, 3, a, b), '\n')
print(torch.addmv(1, m, 1, a, b))

addr allows to perform an outer product of two vectors and add it to a given matrix

  • outer product of two vectors in linear algebra is a matrix

  • e.g., if you have a vector V with m elements (1 dimension) and another vector U with n elements (1 dimension), then outer product of V and U will be a matrix with m × n shape

    • V= [v1, v2, v3…, vm]

    • U = [u1, u2, ……un]

V ⊕ U = A = [ v1u1, v1u2, …. , v1um,’\n’ v2u1, v2u2, …. , v2um,’\n’ ….. vnu1, vnu2, …. , vnum]

In PyTorch, function expects 1st argument as matrix to which we need to add resultant outer product, followed by vectors for which outer product needs to be computed

  • below we create two vectors (a and b) with three elements each, and perform an outer product to create a 3 × 3 matrix, which is then added to another matrix (m)

a = torch.tensor([1.0, 2.0, 3.0])
b = a
m = torch.ones(3,3)
print(f"shapes of a, b, and m tensors are {a.shape}, {b.shape}, and {m.shape}\n")
print(torch.addr(m, a, b), '\n')

m = torch.zeros(3,3)
print(torch.addr(m, a, b), '\n')

4.5 Activation functions

a_tensor  = torch.linspace(-1.0, 1.0, steps=9)
print(f"a_tensor = {a_tensor}")

print(f"using sigmoid function = {torch.sigmoid(a_tensor)}]")
print(f"using tanh    function = {torch.tanh(a_tensor)}]")
print(f"using log1p   function = {torch.log1p(a_tensor)}]")
print(f"using erf     function = {torch.erf(a_tensor)}]")
print(f"using erfinv  function = {torch.erfinv(a_tensor)}]")

print(f"using sigmoid function = {torch.sigmoid(a_tensor)}]")
print(f"using tanh    function = {torch.tanh(a_tensor)}]")
print(f"using log1p   function = {torch.log1p(a_tensor)}]")
print(f"using erf     function = {torch.erf(a_tensor)}]")
print(f"using erfinv  function = {torch.erfinv(a_tensor)}]")

5. Tensors in DL frameworks

5.1 Tensor.device (CPU vs CUDA)

a_tensor = torch.rand(3,4) # 'cpu by default'
print(f'a_tensor = {a_tensor} and its tensor device = {a_tensor.device}\n')

a_tensor = torch.rand(3,4, device='cpu')
print(f'a_tensor = {a_tensor} and its tensor device = {a_tensor.device}\n')

a_tensor = torch.rand(3,4, device='cuda') # cuda for gpu, there is no error if I have GPU
print(f'a_tensor = {a_tensor} and its tensor device = {a_tensor.device}\n')

print(f"PyTorcch version: {torch.__version__}\n")

if torch.cuda.is_available():
    device='cuda'
else:
    device='cpu'
# device = 'cuda' if torch.cuda.is_available() else 'cpu'

print(f"Device: {device}\n")

a_tensor = torch.rand((2,4),device=device)
print(f'a_tensor = {a_tensor} and its tensor device = {a_tensor.device}\n')

b_tensor = torch.rand((2,4),device='cpu')
print(f'b_tensor = {b_tensor} and its tensor device = {b_tensor.device}\n')
# print(a_tensor + b_tensor) # not compatible as is on CPU and one is on GPU

Moving Tensors between CPUs and GPUs

  • Transfer data from CPU to GPU

  • After training, output Tensors are produced in GPU

  • Output data requires preprocessing

  • Some preprocessing libraries don’t support Tensors and expect a Numpy array

  • Numpy supports only data in GPU, and we need to move data from GPU to CPU

# Data from CPU to GPU -- 3 methods

a_tensor = torch.rand((2,4),device='cpu')
print(f'a_tensor = {a_tensor} and its tensor device = {a_tensor.device}\n')

# 1. `Tensor.cuda()`
b_tensor = a_tensor.cuda()
print(f'b_tensor = {b_tensor} and its tensor device = {b_tensor.device}\n')

# 2. `Tensor.to("cuda")`
b_tensor = a_tensor.to("cuda")
print(f'b_tensor = {b_tensor} and its tensor device = {b_tensor.device}\n')

# 3. `Tensor.to("cuda:0")`
b_tensor = a_tensor.to("cuda:0")
print(f'b_tensor = {b_tensor} and its tensor device = {b_tensor.device}\n')

# Data from GPU to CPU -- 2 methods

a_tensor = torch.rand((4), requires_grad=True, device='cuda')
print(f'a_tensor = {a_tensor} and its tensor device = {a_tensor.device}\n')

# 1. `Tensor.cpu()` with `requires_grad=False`
b_tensor = a_tensor.cpu()
print(f'b_tensor = {b_tensor} and its tensor device = {b_tensor.device}\n')

# 2. `Tensor.detach().cpu() with `requires_grad=True`
b_tensor = a_tensor.detach().cpu()
print(f'b_tensor = {b_tensor} and its tensor device = {b_tensor.device}\n')

# tensor to numpy arrays

a_tensor = torch.rand((2,2), requires_grad=True, device='cuda')
print(f'a_tensor = {a_tensor} and its tensor device = {a_tensor.device}\n')

a_array = a_tensor.detach().cpu().numpy()
print(f'a_array = {a_array} and its array device = {a_array.device}\n')

b_tensor = torch.from_numpy(a_array)
print(f'b_tensor = {b_tensor} and its tensor device = {b_tensor.device}\n')