Train model Keras với Callback lưu checkpoint Python

Mô tả

Sử dụng ModelCheckpoint callback để tự động lưu weights sau mỗi epoch (hoặc mỗi N batch) trong quá trình training.

Code ví dụ

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
import numpy as np
import os

def create_model():
    model = keras.Sequential([
        layers.Dense(64, activation='relu', input_shape=(784,)),
        layers.Dense(10, activation='softmax')
    ])
    model.compile(optimizer='adam', 
                  loss='sparse_categorical_crossentropy', 
                  metrics=['accuracy'])
    return model

# Data
x_train = np.random.rand(1000, 784).astype('float32')
y_train = np.random.randint(0, 10, 1000)
x_test = np.random.rand(200, 784).astype('float32')
y_test = np.random.randint(0, 10, 200)

# 1. THIẾT LẬP CHECKPOINT PATH
checkpoint_path = "training_1/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
os.makedirs(checkpoint_dir, exist_ok=True)

# 2. TẠO CALLBACK
cp_callback = keras.callbacks.ModelCheckpoint(
    filepath=checkpoint_path,
    save_weights_only=True,      # Chỉ lưu weights (nhỏ hơn)
    save_best_only=False,        # True: chỉ lưu best theo monitor
    monitor='val_accuracy',      # Metric theo dõi
    mode='max',                  # max cho accuracy, min cho loss
    verbose=1
)

# 3. TRAIN VỚI CALLBACK
model = create_model()
print("Training with checkpoint callback...")
history = model.fit(
    x_train, y_train,
    epochs=10,
    validation_data=(x_test, y_test),
    callbacks=[cp_callback],
    verbose=1
)

# 4. KIỂM TRA FILE CHECKPOINT
print(f"\nCheckpoint files in {checkpoint_dir}:")
for f in os.listdir(checkpoint_dir):
    print(f"  {f}")

# 5. LOAD LATEST VÀ EVALUATE
latest = tf.train.latest_checkpoint(checkpoint_dir)
print(f"\nLatest checkpoint: {latest}")

model2 = create_model()
model2.load_weights(latest)
loss, acc = model2.evaluate(x_test, y_test, verbose=0)
print(f"Restored from checkpoint - Accuracy: {acc:.4f}")

ModelCheckpoint Options quan trọng

Pattern tên file hữu ích

• cp-{epoch:04d}.ckpt: Mỗi epoch một file (cp-0001.ckpt, cp-0002.ckpt...).
• best_model.ckpt với save_best_only=True: Chỉ 1 file best.
• weights_epoch{epoch:02d}_valacc{val_accuracy:.2f}.h5: Thông tin chi tiết trong tên.

Kiểm tra checkpoint files

# Liệt kê
!ls -la checkpoints/

# Xem chi tiết variables
import tensorflow as tf
reader = tf.train.load_checkpoint('checkpoints/cp-0005.ckpt')
print(reader.get_variable_to_shape_map())