diff --git a/sloth/core/commands.py b/sloth/core/commands.py
index 28f6c66..23851c9 100644
--- a/sloth/core/commands.py
+++ b/sloth/core/commands.py
@@ -123,6 +123,7 @@ class AppendFilesCommand(BaseCommand):
             else:
                 logger.debug("Adding image file: %s" % rel_filename)
                 item = self.labeltool.addImageFile(rel_filename)
+            present_filenames.add(rel_filename)
 
             if options['unlabeled']:
                 item.setUnlabeled(True)