What Are the Tensor Operations in TensorFlow and How to Efficiently Process Tensors

Tensors are the core data structures in TensorFlow. Understanding tensor operations is crucial for efficient use of TensorFlow.

Tensor Basics

1. Creating Tensors

python
import tensorflow as tf

# Create tensor from Python list
tensor1 = tf.constant([1, 2, 3, 4])
print(tensor1)  # tf.Tensor([1 2 3 4], shape=(4,), dtype=int32)

# Specify data type
tensor2 = tf.constant([1, 2, 3], dtype=tf.float32)
print(tensor2)  # tf.Tensor([1. 2. 3.], shape=(3,), dtype=float32)

# Create zero tensor
zeros = tf.zeros([3, 4])
print(zeros.shape)  # (3, 4)

# Create ones tensor
ones = tf.ones([2, 3])
print(ones.shape)  # (2, 3)

# Create tensor with specified values
filled = tf.fill([2, 3], 5)
print(filled)  # [[5 5 5] [5 5 5]]

# Create random tensor
random_normal = tf.random.normal([3, 4], mean=0.0, stddev=1.0)
random_uniform = tf.random.uniform([3, 4], minval=0, maxval=1)

# Create sequence tensor
range_tensor = tf.range(0, 10, 2)
print(range_tensor)  # [0 2 4 6 8]

# Create from NumPy array
import numpy as np
numpy_array = np.array([1, 2, 3])
tensor_from_numpy = tf.constant(numpy_array)

2. Tensor Properties

python
# Get tensor shape
tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
print(tensor.shape)  # (2, 3)
print(tf.shape(tensor))  # tf.Tensor([2 3], shape=(2,), dtype=int32)

# Get tensor data type
print(tensor.dtype)  # <dtype: 'int32'>

# Get tensor rank
print(tf.rank(tensor))  # tf.Tensor(2, shape=(), dtype=int32)

# Get tensor size
print(tf.size(tensor))  # tf.Tensor(6, shape=(), dtype=int32)

# Get number of elements
print(tensor.numpy().size)  # 6

Tensor Indexing and Slicing

1. Basic Indexing

python
# Create example tensor
tensor = tf.constant([[1, 2, 3], [4, 5, 6], [7, 8, 9]])

# Get single element
element = tensor[0, 1]  # 2

# Get a row
row = tensor[1, :]  # [4, 5, 6]

# Get a column
col = tensor[:, 1]  # [2, 5, 8]

# Use negative indexing
last_row = tensor[-1, :]  # [7, 8, 9]

2. Slicing Operations

python
# Slicing
tensor = tf.constant([[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]])

# Basic slicing
sliced = tensor[0:2, 1:3]  # [[2, 3], [6, 7]]

# Stride slicing
stepped = tensor[::2, ::2]  # [[1, 3], [9, 11]]

# Simplified indexing
simplified = tensor[1, :]  # [5, 6, 7, 8]

3. Advanced Indexing

python
# Use tf.gather
tensor = tf.constant([[1, 2, 3], [4, 5, 6], [7, 8, 9]])
indices = tf.constant([0, 2])
gathered = tf.gather(tensor, indices)  # [[1, 2, 3], [7, 8, 9]]

# Use tf.gather_nd
indices = tf.constant([[0, 1], [2, 0]])
gathered_nd = tf.gather_nd(tensor, indices)  # [2, 7]

# Use tf.where
condition = tf.constant([[True, False], [False, True]])
values = tf.constant([[1, 2], [3, 4]])
result = tf.where(condition, values, tf.zeros_like(values))
# [[1, 0], [0, 4]]

# Use tf.boolean_mask
mask = tf.constant([True, False, True])
masked = tf.boolean_mask(tensor, mask)  # [[1, 2, 3], [7, 8, 9]]

Tensor Operations

1. Arithmetic Operations

python
# Basic arithmetic operations
a = tf.constant([1, 2, 3])
b = tf.constant([4, 5, 6])

# Addition
add = tf.add(a, b)  # [5, 7, 9]
add = a + b  # [5, 7, 9]

# Subtraction
subtract = tf.subtract(a, b)  # [-3, -3, -3]
subtract = a - b  # [-3, -3, -3]

# Multiplication
multiply = tf.multiply(a, b)  # [4, 10, 18]
multiply = a * b  # [4, 10, 18]

# Division
divide = tf.divide(a, b)  # [0.25, 0.4, 0.5]
divide = a / b  # [0.25, 0.4, 0.5]

# Power
power = tf.pow(a, 2)  # [1, 4, 9]
power = a ** 2  # [1, 4, 9]

# Matrix multiplication
matrix_a = tf.constant([[1, 2], [3, 4]])
matrix_b = tf.constant([[5, 6], [7, 8]])
matmul = tf.matmul(matrix_a, matrix_b)  # [[19, 22], [43, 50]]
matmul = matrix_a @ matrix_b  # [[19, 22], [43, 50]]

2. Mathematical Functions

python
# Trigonometric functions
x = tf.constant([0, np.pi/2, np.pi])
sin = tf.sin(x)  # [0, 1, 0]
cos = tf.cos(x)  # [1, 0, -1]
tan = tf.tan(x)  # [0, inf, 0]

# Exponential and logarithmic
exp = tf.exp(tf.constant([0, 1, 2]))  # [1, 2.718, 7.389]
log = tf.log(tf.constant([1, 2, 3]))  # [0, 0.693, 1.099]
log10 = tf.log(tf.constant([10, 100, 1000])) / tf.log(10.0)  # [1, 2, 3]

# Other mathematical functions
abs = tf.abs(tf.constant([-1, -2, 3]))  # [1, 2, 3]
sqrt = tf.sqrt(tf.constant([1, 4, 9]))  # [1, 2, 3]
square = tf.square(tf.constant([1, 2, 3]))  # [1, 4, 9]
round = tf.round(tf.constant([1.2, 2.7, 3.5]))  # [1, 3, 4]
ceil = tf.ceil(tf.constant([1.2, 2.7, 3.5]))  # [2, 3, 4]
floor = tf.floor(tf.constant([1.2, 2.7, 3.5]))  # [1, 2, 3]

3. Statistical Operations

python
# Create example tensor
tensor = tf.constant([[1, 2, 3], [4, 5, 6], [7, 8, 9]])

# Sum
sum_all = tf.reduce_sum(tensor)  # 45
sum_axis0 = tf.reduce_sum(tensor, axis=0)  # [12, 15, 18]
sum_axis1 = tf.reduce_sum(tensor, axis=1)  # [6, 15, 24]

# Mean
mean_all = tf.reduce_mean(tensor)  # 5.0
mean_axis0 = tf.reduce_mean(tensor, axis=0)  # [4, 5, 6]
mean_axis1 = tf.reduce_mean(tensor, axis=1)  # [2, 5, 8]

# Maximum
max_all = tf.reduce_max(tensor)  # 9
max_axis0 = tf.reduce_max(tensor, axis=0)  # [7, 8, 9]
max_axis1 = tf.reduce_max(tensor, axis=1)  # [3, 6, 9]

# Minimum
min_all = tf.reduce_min(tensor)  # 1
min_axis0 = tf.reduce_min(tensor, axis=0)  # [1, 2, 3]
min_axis1 = tf.reduce_min(tensor, axis=1)  # [1, 4, 7]

# Standard deviation
std = tf.math.reduce_std(tf.cast(tensor, tf.float32))  # 2.582

# Variance
var = tf.math.reduce_variance(tf.cast(tensor, tf.float32))  # 6.667

Tensor Shape Operations

1. Shape Transformation

python
# Reshape
tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
reshaped = tf.reshape(tensor, [3, 2])  # [[1, 2], [3, 4], [5, 6]]

# Flatten
flattened = tf.reshape(tensor, [-1])  # [1, 2, 3, 4, 5, 6]

# Transpose
transposed = tf.transpose(tensor)  # [[1, 4], [2, 5], [3, 6]]

# Squeeze (remove dimensions of size 1)
tensor = tf.constant([[[1], [2], [3]]])
squeezed = tf.squeeze(tensor)  # [1, 2, 3]

# Expand dims (add dimension)
expanded = tf.expand_dims(tensor, axis=0)  # [[[[1], [2], [3]]]]

2. Dimension Operations

python
# Stack
a = tf.constant([1, 2, 3])
b = tf.constant([4, 5, 6])
stacked = tf.stack([a, b], axis=0)  # [[1, 2, 3], [4, 5, 6]]

# Unstack
unstacked = tf.unstack(stacked, axis=0)  # [tf.Tensor([1 2 3]), tf.Tensor([4 5 6])]

# Concat
concat_axis0 = tf.concat([a, b], axis=0)  # [1, 2, 3, 4, 5, 6]

# Split
tensor = tf.constant([[1, 2, 3], [4, 5, 6]])
split = tf.split(tensor, 2, axis=0)  # [tf.Tensor([[1 2 3]]), tf.Tensor([[4 5 6]])]

# Tile (repeat tensor)
tensor = tf.constant([[1, 2], [3, 4]])
tiled = tf.tile(tensor, [2, 3])  # [[1, 2, 1, 2, 1, 2], [3, 4, 3, 4, 3, 4], [1, 2, 1, 2, 1, 2], [3, 4, 3, 4, 3, 4]]

# Repeat (repeat elements)
tensor = tf.constant([[1, 2], [3, 4]])
repeated = tf.repeat(tensor, repeats=2, axis=0)  # [[1, 2], [1, 2], [3, 4], [3, 4]]

3. Padding and Cropping

python
# Pad
tensor = tf.constant([[1, 2], [3, 4]])
padded = tf.pad(tensor, [[1, 1], [1, 1]], mode='CONSTANT')
# [[0, 0, 0, 0], [0, 1, 2, 0], [0, 3, 4, 0], [0, 0, 0, 0]]

# Crop
cropped = tf.image.crop_to_bounding_box(tensor, 0, 0, 1, 1)  # [[1]]

# Resize (mainly for images)
image = tf.random.uniform([100, 100, 3])
resized = tf.image.resize(image, [50, 50])  # [50, 50, 3]

Tensor Broadcasting

python
# Broadcasting mechanism
a = tf.constant([[1, 2, 3], [4, 5, 6]])  # shape (2, 3)
b = tf.constant([1, 2, 3])  # shape (3,)

# Automatic broadcasting
result = a + b  # shape (2, 3)
# [[2, 4, 6], [5, 7, 9]]

# Explicit broadcasting
a = tf.constant([[1], [2], [3]])  # shape (3, 1)
b = tf.constant([1, 2, 3])  # shape (3,)
result = tf.broadcast_to(a, [3, 3]) + tf.broadcast_to(b, [3, 3])
# [[2, 3, 4], [3, 4, 5], [4, 5, 6]]

# Check if broadcasting is possible
a = tf.constant([1, 2, 3])
b = tf.constant([1, 2])
try:
    result = a + b
except tf.errors.InvalidArgumentError as e:
    print("Broadcasting not possible:", e)

Tensor Data Type Conversion

python
# Type conversion
int_tensor = tf.constant([1, 2, 3])
float_tensor = tf.cast(int_tensor, tf.float32)  # [1.0, 2.0, 3.0]

# Check type
print(int_tensor.dtype)  # <dtype: 'int32'>
print(float_tensor.dtype)  # <dtype: 'float32'>

# Convert to NumPy array
numpy_array = int_tensor.numpy()
print(type(numpy_array))  # <class 'numpy.ndarray'>

# Create tensor from NumPy array
new_tensor = tf.constant(numpy_array)

Tensor Comparison

python
# Equality comparison
a = tf.constant([1, 2, 3])
b = tf.constant([1, 2, 4])
equal = tf.equal(a, b)  # [True, True, False]
not_equal = tf.not_equal(a, b)  # [False, False, True]

# Size comparison
greater = tf.greater(a, b)  # [False, False, False]
less = tf.less(a, b)  # [False, False, True]
greater_equal = tf.greater_equal(a, b)  # [True, True, False]
less_equal = tf.less_equal(a, b)  # [True, True, True]

# Element-wise max and min
maximum = tf.maximum(a, b)  # [1, 2, 4]
minimum = tf.minimum(a, b)  # [1, 2, 3]

Tensor Sorting

python
# Sorting
tensor = tf.constant([3, 1, 4, 1, 5, 9, 2, 6])
sorted = tf.sort(tensor)  # [1, 1, 2, 3, 4, 5, 6, 9]

# Get sorted indices
indices = tf.argsort(tensor)  # [1, 3, 6, 0, 2, 4, 7, 5]

# Sort by value
values, indices = tf.nn.top_k(tensor, k=3)
# values: [9, 6, 5]
# indices: [5, 7, 4]

# Sort 2D tensor
tensor_2d = tf.constant([[3, 1, 4], [1, 5, 9], [2, 6, 5]])
sorted_2d = tf.sort(tensor_2d, axis=1)
# [[1, 3, 4], [1, 5, 9], [2, 5, 6]]

Tensor Concatenation and Splitting

python
# Concatenation
a = tf.constant([[1, 2], [3, 4]])
b = tf.constant([[5, 6], [7, 8]])

# Concatenate along axis 0
concat_0 = tf.concat([a, b], axis=0)
# [[1, 2], [3, 4], [5, 6], [7, 8]]

# Concatenate along axis 1
concat_1 = tf.concat([a, b], axis=1)
# [[1, 2, 5, 6], [3, 4, 7, 8]]

# Splitting
tensor = tf.constant([[1, 2, 3], [4, 5, 6], [7, 8, 9]])

# Uniform split
split_0 = tf.split(tensor, 3, axis=0)
# [tf.Tensor([[1 2 3]]), tf.Tensor([[4 5 6]]), tf.Tensor([[7 8 9]])]

# Non-uniform split
split_1 = tf.split(tensor, [1, 2], axis=0)
# [tf.Tensor([[1 2 3]]), tf.Tensor([[4 5 6] [7 8 9]])]

Tensor Masking and Conditional Operations

python
# Boolean masking
tensor = tf.constant([1, 2, 3, 4, 5])
mask = tf.constant([True, False, True, False, True])
masked = tf.boolean_mask(tensor, mask)  # [1, 3, 5]

# Conditional selection
condition = tf.constant([True, False, True])
x = tf.constant([1, 2, 3])
y = tf.constant([4, 5, 6])
result = tf.where(condition, x, y)  # [1, 5, 3]

# Conditional masking
tensor = tf.constant([1, 2, 3, 4, 5])
mask = tensor > 3
result = tf.boolean_mask(tensor, mask)  # [4, 5]

Tensor Reduction Operations

python
# Reduction operations
tensor = tf.constant([[1, 2, 3], [4, 5, 6], [7, 8, 9]])

# Reduce along axis
sum_axis0 = tf.reduce_sum(tensor, axis=0)  # [12, 15, 18]
sum_axis1 = tf.reduce_sum(tensor, axis=1)  # [6, 15, 24]

# Keep dimensions
sum_keepdims = tf.reduce_sum(tensor, axis=0, keepdims=True)
# [[12, 15, 18]]

# Multi-axis reduction
sum_multi = tf.reduce_sum(tensor, axis=[0, 1])  # 45

# Specific reductions
prod = tf.reduce_prod(tensor)  # 362880
all_true = tf.reduce_all(tensor > 0)  # True
any_true = tf.reduce_any(tensor > 5)  # True

Efficient Tensor Operation Tips

1. Use Vectorized Operations

python
# Inefficient way
result = []
for i in range(1000):
    result.append(i * 2)

# Efficient way
tensor = tf.range(1000)
result = tensor * 2

2. Batch Processing

python
# Batch matrix multiplication
batch_size = 32
matrices_a = tf.random.normal([batch_size, 100, 100])
matrices_b = tf.random.normal([batch_size, 100, 100])
result = tf.matmul(matrices_a, matrices_b)  # [32, 100, 100]

3. Use tf.function for Acceleration

python
@tf.function
def fast_operation(x):
    return tf.reduce_sum(x * 2)

# First call compiles, subsequent calls are faster
result = fast_operation(tf.random.normal([1000, 1000]))

4. Avoid Unnecessary Copies

python
# Use tf.identity to avoid copying
a = tf.constant([1, 2, 3])
b = tf.identity(a)  # Doesn't create new tensor, just reference

5. Use Appropriate Devices

python
# Use GPU
with tf.device('/GPU:0'):
    a = tf.random.normal([1000, 1000])
    b = tf.random.normal([1000, 1000])
    c = tf.matmul(a, b)

# Use CPU
with tf.device('/CPU:0'):
    a = tf.constant([1, 2, 3])
    b = tf.constant([4, 5, 6])
    c = a + b

6. Memory Optimization

python
# Use tf.data.Dataset for efficient data loading
dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
dataset = dataset.batch(32).prefetch(tf.data.AUTOTUNE)

# Use tf.Variable to avoid repeated creation
var = tf.Variable(tf.zeros([1000, 1000]))
var.assign(tf.random.normal([1000, 1000]))

Tensor Operation Performance Optimization

1. XLA Compilation

python
# Use XLA compilation for acceleration
@tf.function(experimental_compile=True)
def xla_compatible_function(x):
    return tf.reduce_sum(x ** 2)

2. Mixed Precision

python
from tensorflow.keras import mixed_precision

# Enable mixed precision
policy = mixed_precision.Policy('mixed_float16')
mixed_precision.set_global_policy(policy)

# Tensors will automatically use float16 for computation
a = tf.random.normal([1000, 1000])
b = tf.random.normal([1000, 1000])
c = tf.matmul(a, b)  # Computed using float16

3. Parallelization

python
# Use parallel mapping
dataset = tf.data.Dataset.range(1000)
dataset = dataset.map(lambda x: x * 2, num_parallel_calls=tf.data.AUTOTUNE)

Summary

TensorFlow's tensor operations provide powerful data processing capabilities:

• Tensor Creation: Multiple ways to create tensors
• Indexing and Slicing: Flexible indexing and slicing operations
• Tensor Operations: Rich arithmetic and mathematical operations
• Shape Operations: Flexible shape transformations
• Broadcasting: Automatically handles tensors of different shapes
• Efficient Operations: Vectorization, batch processing, device selection, etc.

Mastering these tensor operations will help you use TensorFlow more efficiently for deep learning development.