diff --git a/python/ray/tune/BUILD b/python/ray/tune/BUILD
index 6cbb391a9..742997fc3 100644
--- a/python/ray/tune/BUILD
+++ b/python/ray/tune/BUILD
@@ -488,7 +488,7 @@ py_test(
     size = "large",
     srcs = ["examples/pbt_transformers/pbt_transformers.py"],
     deps = [":tune_lib"],
-    tags = ["exclusive", "example"],
+    tags = ["flaky", "exclusive", "example"],
     args = ["--smoke-test"]
 )
 
diff --git a/python/ray/tune/examples/pbt_transformers/pbt_transformers.py b/python/ray/tune/examples/pbt_transformers/pbt_transformers.py
index 0e739b524..0e827da3f 100644
--- a/python/ray/tune/examples/pbt_transformers/pbt_transformers.py
+++ b/python/ray/tune/examples/pbt_transformers/pbt_transformers.py
@@ -131,7 +131,7 @@ def tune_transformer(num_samples=8,
 
     # Change these as needed.
     model_name = "bert-base-uncased" if not smoke_test \
-        else "distilbert-base-uncased"
+        else "sshleifer/tiny-distilroberta-base"
     task_name = "rte"
 
     task_data_dir = os.path.join(data_dir, task_name.upper())