diff --git a/pytorch_lightning/models/trainer.py b/pytorch_lightning/models/trainer.py
index ab1d8565..a6d6fe6d 100644
--- a/pytorch_lightning/models/trainer.py
+++ b/pytorch_lightning/models/trainer.py
@@ -444,11 +444,7 @@ class Trainer(TrainerIO):
 
             # run through amp wrapper
             if self.use_amp:
-                # An example
-                model, optimizers = amp.initialize(
-                    model, self.optimizers, opt_level=self.amp_level,
-                )
-                self.optimizers = optimizers
+                raise MisconfigurationException('amp + cpu is not supported. Please use a GPU option')
 
             self.__run_pretrain_routine(model)
 
diff --git a/tests/test_models.py b/tests/test_models.py
index 15a41e9c..0e933382 100644
--- a/tests/test_models.py
+++ b/tests/test_models.py
@@ -40,7 +40,8 @@ def test_cpu_model_with_amp():
 
     model, hparams = get_model()
 
-    run_gpu_model_test(trainer_options, model, hparams, on_gpu=False)
+    with pytest.raises(MisconfigurationException):
+        run_gpu_model_test(trainer_options, model, hparams, on_gpu=False)
 
 
 def test_amp_gpu_ddp_slurm_managed():