diff --git a/doc/api/index.rst b/doc/api/index.rst index 8d6b549..73101f0 100644 --- a/doc/api/index.rst +++ b/doc/api/index.rst @@ -7,6 +7,7 @@ API Reference .. toctree:: :maxdepth: 1 + labeltool containers model scene diff --git a/doc/api/labeltool.rst b/doc/api/labeltool.rst new file mode 100644 index 0000000..830235f --- /dev/null +++ b/doc/api/labeltool.rst @@ -0,0 +1,14 @@ +========= +Labeltool +========= + +The labeltool object is the main object that hold most of the current state of +the label tool. + +It provides the following API: + +.. module:: sloth.core.labeltool + +.. autoclass:: LabelTool + :members: + diff --git a/doc/configuration.rst b/doc/configuration.rst index 0e27c6f..abcce2d 100644 --- a/doc/configuration.rst +++ b/doc/configuration.rst @@ -17,78 +17,94 @@ LABELS Default:: ( - ("Rect", {"type": "rect"}), - ("Point", {"type": "point"}), - ("Polygon", {"type": "polygon"}), + { + 'attributes': { + 'type': 'rect', + }, + 'inserter': 'sloth.items.RectItemInserter', + 'item': 'sloth.items.RectItem', + 'hotkey': 'r', + 'text': 'Rectangle', + }, + { + 'attributes': { + 'type': 'point', + }, + 'inserter': 'sloth.items.PointItemInserter', + 'item': 'sloth.items.PointItem', + 'hotkey': 'p', + 'text': 'Point', + }, ) -List of labels. This will be used to construct the button area from which the user can select to be created -labels. The second tuple entry is expected to be a python dictionary, which contains at least the key `type`. -All other keys are optional, but are directly used for the newly created label. A value can also be a list. -In this case, the button area displays another list of options for the key as defined in the list. Example:: +``LABELS`` is a tuple/list of dictionaries. Each dictionary describe how one +annotation type is visualized, newly inserted and modified. Let's go over the +different keys of the dictionary in detail: - ( - ("Rect", {"type": "rect", "class": "head", "id": ["Martin", "Mika", "Boris"]}), - ) +* ``text``: This is a text that describes the label type, and will be + displayed to the user in the GUI. -Note two things here. First, the comma at the end of the first tuple is mandatory. Otherwise the outer tuple -will not be recognized as one (it will be only parentheses around an object, which will alone not be translated -into a tuple object. Second, the key `head` does not contain a list as value. That means, that this key-value -pair will be used directly as such in a newly created label. For the key `id` the user can choose from the -given list, and only the chosen value will be used for the newly created label. +* ``item`` specifies which class is responsible for visualizing the annotation. + For the first annotation type we chose to use the predefined + ``sloth.items.RectItem`` class, which will draw a rectangle as given by the + coordinates in the annotation. Sloth comes with several predefined + visualization classes, such as ``sloth.items.RectItem`` and + ``sloth.items.PointItem`` (see :ref:`items` for a full list). However, it is + also very easy to define your own visualization class (see :ref:`items`). -.. _ITEMS: +* ``inserter`` specifies which class is responsible for creating new + annotations based on user input. When the user enters insert mode with a + given label type, the corresponding inserter is instantiated and captures all + user input for the creation of a new annotation. The inserter is passed the + current state of the button area. -ITEMS ------ +* ``attributes`` has three functions: + 1. It defines how a new annotation can be initialized. Fixed + key-value pairs are used directly. If the value is a list of items, the + user can choose interactively which one of the values he wants to use for + a new label. The current state is then passed to the inserter. -Default:: + 2. It defines how a existing annotations can be edited. Fixed + key-value are not allowed to be edited. If the value is a list of items, the + user can choose interactively between the values for the corresponding key. + The annotation is then updated accordingly. - { - "rect": 'items.RectItem', - "point": 'items.PointItem', - "polygon": 'items.PolygonItem', - } + 3. It defines how to match an existing annotation to one of the entries in ``LABELS``. + Sloth uses a soft matching based on the two keys ``class`` and ``type``. It checks + each item in ``LABELS`` starting from the beginning and stops if it finds the first + match. An entry matches an annotation if: -Mapping from `type` to the visualization item. The values need to be python callables that create -a new visualization item. They don't neccessarily need to be subclasses of `AnnotationGraphicsItem`. -Nevertheless, the constructor of any subclass of `AnnotationGraphicItem` is of course a python callable -that creates a new visualization item. + * the values for both keys match, or + * the value for one of keys matches and the other key is not present in + either ``attributes`` or the annotation. + + +Note that the comma at the end of the first tuple is mandatory. Otherwise the +outer tuple will not be recognized as one (it will be only parentheses around +an object, which will alone not be translated into a tuple object. This +applies similarly to all tuple/list-type settings. .. _HOTKEYS: HOTKEYS ------- -Default:: ``()`` (Empty tuple) - -Hold a list of hotkeys for the label inserting. Example:: - - ( - ("Point", "", "", "p"), - ("Rect", "Id", "Martin", "Ctrl+M"), - ) - -.. todo:: This sucks! Please come up with a better way to reference the labels. Also it might be - interesting to be able to assign hotkeys to other tasks, such as "copy all labels from the previous frame" - May merge with LABELS, and assign hotkeys there! - -.. _INSERTERS: - -INSERTERS ---------- - Default:: - { - 'rect': 'items.RectInserter', - 'point': 'items.PointInserter', - 'polygon': 'items.PolygonInserter', - } + ( + ('PgDown', lambda lt: lt.gotoNext(), 'Next image/frame'), + ('PgUp', lambda lt: lt.gotoPrevious(), 'Previous image/frame'), + ('Tab', lambda lt: lt.selectNextAnnotation(), 'Select next annotation'), + ('Shift+Tab', lambda lt: lt.selectPreviousAnnotation(), 'Select previous annotation'), + ('Del', lambda lt: lt.deleteSelectedAnnotations(), 'Delete selected annotations'), + ('ESC', lambda lt: lt.exitInsertMode(), 'Exit insert mode'), + ) -Defines a mapping of which inserter should be used for interactively inserting a new label -into the image. The default inserters allow to draw the respective shape. Read more -about how to write your own inserter in :ref:`Inserters`. +Defines global keyboard shortcuts. Each hotkey is defined by a tuple with at +least 2 entries, where the first entry is the hotkey (sequence), and the second +entry is the function that is called. The function should expect a single +parameter, the labeltool object. The optional third entry -- if present -- is +expected to be a string describing the action. .. _CONTAINERS: @@ -103,9 +119,9 @@ Default:: '*.pickle': 'annotations.container.PickleContainer', } -Defines a mapping of which container should be used for loading a label file matching the given filename pattern. -This can of course also be a user defined container. You can also define the class directly (instead -of a module path):: +Defines a mapping of which container should be used for loading a label file +matching the given filename pattern. This can of course also be a user defined +container. You can also define the class directly (instead of a module path):: { '*.foo': MyFooContainer @@ -119,3 +135,21 @@ PLUGINS Did not think to much about this yet. This is rather for v2.0. Could image to be able to define some kind of plugin that might do some preprocessing on an image, e.g. detect all faces and convert them into labels. + +Extending default values +======================== + +In the usual case one overrides the default when defining a setting +variable. In order to extend the default configuration and avoid overriding +the default values, you can first import the default configuration and then +append your custom mappings (remember that the configuration is a python +module, you can basically execute any valid python code):: + + from conf.default_config import LABELS + + MYLABLES = ({ + ... + }) + + LABELS += MYLABELS + diff --git a/doc/containers.rst b/doc/containers.rst index f04eea3..2062af1 100644 --- a/doc/containers.rst +++ b/doc/containers.rst @@ -5,6 +5,94 @@ Containers ========== Annotation containers provide functions for loading and saving labels. You can -supply custom containers to support specific label formats. +write custom containers to support specific label formats. + +Container Interface +=================== + +A container is expected to implement (at least) these five functions: + +.. py:function:: load(self, filename) + + Loads and returns the annotations in file ``filename``. + +.. py:function:: save(self, annotations, filename) + + Writes the given annotations to file ``filename``. + +.. py:function:: filename(self) + + Returns the current filename. + +.. py:function:: loadImage(self, filename) + + Loads and returns the image referenced to by filename + +.. py:function:: loadFrame(self, filename, frame_number) + + Load the video referenced to by the filename, and return frame + ``frame_number``. + +The container base class ``AnnotationContainer`` provides default +implementations for all five function. It however deferes the +parsing and serialization of the labels from/to disk to the to functions + +.. py:function:: parseFromFile(self, filename) + +and + +.. py:function:: serializeToFile(self, filename, annotations + +respectively. If you subclass AnnotationContainer, make sure to +provide implementations for those two functions. + + +Default Containers +================== + +A few containers are included in sloth. They can be found in the module +``sloth.annotations.container``. In the default configuration, these +containers are included for their respective default filename patter. + +JsonContainer +------------- + +Default pattern: ``*.json`` + +Writes and reads annotations in JSON format (needs the python module ``json`` +to be installed). + + +YamlContainer +------------- + +Default pattern: ``*.yaml`` + +Writes and reads annotations in YAML format (needs the python module ``yaml`` +to be installed). + +PickleContainer +------------- + +Default pattern: ``*.pickle`` + +Writes and reads annotations in pickle format (needs the python module ``pickle`` +or ``cPickle`` to be installed, ``cPickle`` is more performant). + +FileNameListContainer +--------------------- + +Default pattern: ``*.sloth-init`` + +A simple container that reads one image filename per line. No annotations +are supported. This container can be used for example for initializing +a labeling session. After adding labels, another container should be +used for saving though, otherwise the labels will be lost. (write support +not implemented yet anyway) + +FeretContainer +------------- + +Reads annotations in the Feret format (no write support implemented yet). +This container is not included in the default configuration. -.. todo:: write diff --git a/doc/first_steps.rst b/doc/first_steps.rst index e7e72ea..4c0d603 100644 --- a/doc/first_steps.rst +++ b/doc/first_steps.rst @@ -4,14 +4,16 @@ First Steps =========== -In this section, you will learn with a simple example, how to load labels and write a simple configuration file. -The full configuration options will be covered in the next section :doc:`configuration`. +In this section, you will learn with a simple example, how to load labels and +write a simple configuration file. The full configuration options will be +covered in the next section :doc:`configuration`. Using the default configuration =============================== -The easiest way to start sloth is by using a supported label format and supported label types only. In this case -we just need to start sloth and supply the label file on the command line:: +The easiest way to start sloth is by using a supported label format and +supported label types only. In this case we just need to start sloth and +supply the label file on the command line:: sloth examples/example1_labels.json @@ -51,75 +53,106 @@ Let's take look at the example label file:: } ] -We have labeled two images, with two rectangles in image1 and one point in image 2. Since we launched -sloth without a custom configuration, the standard visualizations for ``rect`` and ``point`` will be used. Sloth -displays two rectangles at the labeled positions in image1, and a point in image2. +We have labeled two images, with two rectangles in image1 and one point in +image 2. Since we launched sloth without a custom configuration, the standard +visualizations for ``rect`` and ``point`` will be used. Sloth displays two +rectangles at the labeled positions in image1, and a point in image2. + +Adding and editing annotation in the GUI +======================================== + +TODO Writing a custom configuration ============================== -The configuration file is a python module where the module-level variables represent the settings. The -most important variables are +The configuration file is a python module where the module-level variables +represent the settings. The most important variable is -* :ref:`ITEMS`: This defines how a given label is visualized by the label tool. -* :ref:`LABELS`: This defines *which* new labels can be created interactively by the user. -* :ref:`INSERTERS`: This defines *how* new labels are created by the user. +* :ref:`LABELS`: This defines how sloth will display annotations and how the + user can insert new ones. We start with a quick example:: - ITEMS = { - 'rect': 'items.RectItem', - 'point': 'items.PointItem', - 'bbox': 'items.RectItem', - } - LABELS = ( - ("Rect", {"type": "rect", - "class": "head", - "id": ["Martin", "Mika"]}), - ("Bounding Box", {"type": "bbox", - "class": "body", - "id": ["Martin", "Mika"]}), + {"attributes": {"type": "rect", + "class": "head", + "id": ["Martin", "Mika"]}, + "item": "sloth.items.RectItem", + "inserter": "sloth.items.RectItemInserter", + "text: "Head" + }, + + {"attributes": {"type": "point", + "class": "left_eye", + "id": ["Martin", "Mika"]}, + "item": "sloth.items.PointItem", + "inserter": "sloth.items.PointItemInserter", + "text: "Left Eye" + }, + + {"attributes": {"type": "point", + "class": "right_eye", + "id": ["Martin", "Mika"]}, + "item": "sloth.items.PointItem", + "inserter": "sloth.items.PointItemInserter", + "text: "Right Eye" + }, ) -In ``ITEMS`` we specify that all labels of type ``rect`` will be visualized by the class ``items.RectItem`` -(which is one of the predefined visualization items that comes with the label tool). All labels of type -``point`` will be visualized by ``items.PointItem``. Note that we can use any type basically. The type -``bbox`` will also be visualized by a ``items.RectItem``. +``LABELS`` is a tuple/list of dictionaries. Each dictionary describe how one +annotation type is visualized, newly inserted and modified. Let's go over the +different keys of the dictionary in detail: -In ``LABELS`` we defined which `new` labels the user can create with the label tool. The variable is -expected to be a list/tuple of tuples. Each of the inner tuples contains first a description of the -label (this will be on the button displayed to the user), and the a description of the label to be -created. In our case, we create a label of type ``rect`` if the user hits the ``Rect`` button. Further, -the newly created label will have the class ``head`` (which is fixed), and the user can choose between -one of the ids from the given list. +* ``text``: This is a text that describes the label type, and will be + displayed to the user in the GUI. -Similarly, the user now can create Bounding Box labels of type ``bbox`` with class ``body``. +* ``item`` specifies which class is responsible for visualizing the annotation. + For the first annotation type we chose to use the predefined + ``sloth.items.RectItem`` class, which will draw a rectangle as given by the + coordinates in the annotation. Sloth comes with several predefined + visualization classes, such as ``sloth.items.RectItem`` and + ``sloth.items.PointItem`` (see :ref:`items` for a full list). However, it is + also very easy to define your own visualization class (see :ref:`items`). -There is a difference between the visualization items and the way the labels are created by the user -interactively. For example, the label tool does *not* know out of the box how to create a label of -type ``bbox``. We have to explicitly specify how to insert this type. We can do this by setting -the ``INSERTERS`` variable:: +* ``inserter`` specifies which class is responsible for creating new + annotations based on user input. When the user enters insert mode with a + given label type, the corresponding inserter is instantiated and captures all + user input for the creation of a new annotation. The inserter is passed the + current state of the button area. - INSERTERS = { - 'rect': 'items.inserters.RectItemInserter', - 'bbox': 'items.inserters.RectItemInserter', - } +* ``attributes`` has three functions: + 1. It defines how a new annotation can be initialized. Fixed + key-value pairs are used directly. If the value is a list of items, the + user can choose interactively which one of the values he wants to use for + a new label. The current state is then passed to the inserter. -The ``RectItemInserter`` lets the user draw a rectangle with the mouse, and then sets the ``x``, -``y``, ``width`` and ``height`` members of the label accordingly. By mapping the type ``bbox`` -to ``RectItemInserter``, the user will be able to draw a rectangle each time a new Bounding Box -label is created. Note that we also have to add the ``RectItemInserter`` for the type ``rect`` -as well (which would also be in the default configuration) due to the fact that we override -the ``INSERTERS`` variable completely. Otherwise the label tool would not know anymore, how -to insert labels of type ``rect``. + 2. It defines how a existing annotations can be edited. Fixed + key-value are not allowed to be edited. If the value is a list of items, the + user can choose interactively between the values for the corresponding key. + The annotation is then updated accordingly. -In order to extend the default configuration and avoid overriding the default values, you can -first import the default configuration and then append your custom mappings (remember that -the configuration is a python module, you can basically execute any valid python code):: + 3. It defines how to match an existing annotation to one of the entries in ``LABELS``. + Sloth uses a soft matching based on the two keys ``class`` and ``type``. It checks + each item in ``LABELS`` starting from the beginning and stops if it finds the first + match. An entry matches an annotation if: - from conf.default_configuration import INSERTERS - INSERTERS['bbox'] = 'items.inserters.RectItemInserter' + * the values for both keys match, or + * the value for one of keys matches and the other key is not present in + either ``attributes`` or the annotation. + +You need to save your custom configuration in a file ending with ".py". To use it +pass it to sloth using the ``--config`` command line parameter:: + + sloth --config myconfig.py examples/example1_labels.json + +You can now start labeling head locations and eye positions. You'll see that for each +depending on the chosen annotation, you can either insert a rectangle (this is internally +done by the ``RectItemInserter``) or points (using the ``PointItemInserter``). For +each annotation you can choose an identity between the two supplied options. + +Next steps +========== You can now continue by reading about :doc:`all available configuration options `, how to write your own :doc:`visualization items ` or how to write :doc:`custom inserters `. diff --git a/sloth/annotations/container.py b/sloth/annotations/container.py index c34d6dd..8757a96 100644 --- a/sloth/annotations/container.py +++ b/sloth/annotations/container.py @@ -1,7 +1,8 @@ import os import fnmatch import time -from sloth.core.exceptions import ImproperlyConfigured, NotImplementedException, InvalidArgumentException +from sloth.core.exceptions import \ + ImproperlyConfigured, NotImplementedException, InvalidArgumentException from sloth.core.utils import import_callable try: import cPickle as pickle @@ -19,6 +20,7 @@ import okapy import logging LOG = logging.getLogger(__name__) + class AnnotationContainerFactory: def __init__(self, containers): """ @@ -131,22 +133,23 @@ class AnnotationContainer: def loadImage(self, filename): """ - Load the image referenced to by the filename. In the default - implementation this will try to load the image from a path - relative to the label files directory. + Load and return the image referenced to by the filename. In the + default implementation this will try to load the image from a path + relative to the label file's directory. """ fullpath = self._fullpath(filename) return okapy.loadImage(fullpath) - def loadVideo(self, filename): + def loadFrame(self, filename, frame_number): """ - Load the video referenced to by the filename. In the default - implementation this will try to load the video from a path - relative to the label files directory. + Load the video referenced to by the filename, and return frame + ``frame_number``. In the default implementation this will try to load + the video from a path relative to the label files directory. """ fullpath = self._fullpath(filename) #TODO load video + class PickleContainer(AnnotationContainer): """ Simple container which pickles the annotations to disk. @@ -233,7 +236,9 @@ class FileNameListContainer(AnnotationContainer): return annotations def serializeToFile(self, filename, annotations): - raise NotImplemented("FileNameListContainer.save() is not implemented yet.") + raise NotImplemented( + "FileNameListContainer.save() is not implemented yet.") + class FeretContainer(AnnotationContainer): """