Skip to content

Publish

NODDCOCODataset

Bases: object

Class for validating and uploading annotations in COCO format.

Parameters:

Name Type Description Default
coco

a COCO-format file

 required
dataset_root str

the root url path for datasets, including the bucket name, etc. consider computing with dataset_path function.

 required
Source code in pynoddgcs/publish.py 
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
class NODDCOCODataset(object):
    """
    Class for validating and uploading annotations in COCO format.

    Parameters
    ----------
    coco: pycocotools.coco.COCO
        a COCO-format file
    dataset_root: str
        the root url path for datasets, including the bucket name, etc.
        consider computing with `dataset_path` function.
    """

    def __init__(self, coco_file: str, dataset_root: str, bucket:str): 
        self.coco_file = coco_file
        self.coco = pycocotools.coco.COCO(coco_file)
        self.coco_root = os.path.split(self.coco_file)[0]
        self.bucket = bucket
        self.relative_gcs_path = dataset_root
        self.gcs = GCS()

    def compute_urls(self):
        """
        Compute and update the urls within this COCO file to reflect the
        expected location of the files within GCS
        """
        for i, image in self.coco.imgs.items():
            # discard the drive letter, if present
            file_name = os.path.splitdrive(image['file_name'])[1]
            splitfile = split_filename(file_name)
            image['coco_url'] = join_urlpath(
                GCS_ROOT,
                self.bucket,
                self.relative_gcs_path, 
                *splitfile
            )

    def unnest_filenames(self, file_separator = '_'):
        """
        pycocotools doesn't appreciated "nested" `file_name attributes.
        To this end, we replace the `file_name` attribute to be "unnested",
        by simply replacing instances of '/' with some other separator.

        Note that the URL will still potentially contain nested paths.
        This method is idempotent if the file_name attribute is unnested.

        Parameters
        ----------
        file_separator: str
            the string we use to replace '/' in the `file_name`
        """
        # COCO file_names should not be nested
        for i, image in self.coco.imgs.items():
            # discard the drive letter, if present
            file_name = os.path.splitdrive(image['file_name'])[1]
            splitfile = split_filename(file_name)
            image['file_name'] = file_separator.join(splitfile)

    def upload_images(self):
        """
        Upload the images in this COCO metadata file to GCS.
        The files should be located at the location specified with 
        the `file_name` attribute, either absolute or relative to 
        the location of the COCO file.
        """
        for i, image in self.coco.imgs.items():
            print(image['file_name'])
            # discard the drive letter, if present
            file_name = os.path.splitdrive(image['file_name'])[1]
            splitfile = split_filename(file_name)
            destination = join_urlpath(
                self.relative_gcs_path, *splitfile
            )
            if os.path.isabs(image['file_name']):
                source = image['file_name']
            else:
                source = os.path.join(self.coco_root, image['file_name'])
            self.gcs.upload(self.bucket, source, destination)

    def upload_coco(self):
        """
        Upload the COCO file, but adjusted so that file urls point to files
        in the GCS bucket.
        """
        self.compute_urls()
        self.unnest_filenames()
        newcoco = json.dumps(self.coco.dataset)
        destination = join_urlpath(
            self.relative_gcs_path, 'annotations.json'
        )
        self.gcs.upload_string(self.bucket, newcoco, destination)

    def upload(self):
        """
        Upload this dataset, first the images, then the adjusted COCO file.
        """
        print("uploading images")
        self.upload_images()
        print("uploading coco file")
        self.upload_coco()

compute_urls()

Compute and update the urls within this COCO file to reflect the expected location of the files within GCS

Source code in pynoddgcs/publish.py 
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def compute_urls(self):
    """
    Compute and update the urls within this COCO file to reflect the
    expected location of the files within GCS
    """
    for i, image in self.coco.imgs.items():
        # discard the drive letter, if present
        file_name = os.path.splitdrive(image['file_name'])[1]
        splitfile = split_filename(file_name)
        image['coco_url'] = join_urlpath(
            GCS_ROOT,
            self.bucket,
            self.relative_gcs_path, 
            *splitfile
        )

unnest_filenames(file_separator='_')

pycocotools doesn't appreciated "nested" file_name attributes. To this end, we replace thefile_name` attribute to be "unnested", by simply replacing instances of '/' with some other separator.

Note that the URL will still potentially contain nested paths. This method is idempotent if the file_name attribute is unnested.

Parameters:

Name Type Description Default
file_separator

the string we use to replace '/' in the file_name

 '_'
Source code in pynoddgcs/publish.py 
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
def unnest_filenames(self, file_separator = '_'):
    """
    pycocotools doesn't appreciated "nested" `file_name attributes.
    To this end, we replace the `file_name` attribute to be "unnested",
    by simply replacing instances of '/' with some other separator.

    Note that the URL will still potentially contain nested paths.
    This method is idempotent if the file_name attribute is unnested.

    Parameters
    ----------
    file_separator: str
        the string we use to replace '/' in the `file_name`
    """
    # COCO file_names should not be nested
    for i, image in self.coco.imgs.items():
        # discard the drive letter, if present
        file_name = os.path.splitdrive(image['file_name'])[1]
        splitfile = split_filename(file_name)
        image['file_name'] = file_separator.join(splitfile)

upload()

Upload this dataset, first the images, then the adjusted COCO file.

Source code in pynoddgcs/publish.py 
103
104
105
106
107
108
109
110
def upload(self):
    """
    Upload this dataset, first the images, then the adjusted COCO file.
    """
    print("uploading images")
    self.upload_images()
    print("uploading coco file")
    self.upload_coco()

upload_coco()

Upload the COCO file, but adjusted so that file urls point to files in the GCS bucket.

Source code in pynoddgcs/publish.py 
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
def upload_coco(self):
    """
    Upload the COCO file, but adjusted so that file urls point to files
    in the GCS bucket.
    """
    self.compute_urls()
    self.unnest_filenames()
    newcoco = json.dumps(self.coco.dataset)
    destination = join_urlpath(
        self.relative_gcs_path, 'annotations.json'
    )
    self.gcs.upload_string(self.bucket, newcoco, destination)

upload_images()

Upload the images in this COCO metadata file to GCS. The files should be located at the location specified with the file_name attribute, either absolute or relative to the location of the COCO file.

Source code in pynoddgcs/publish.py 
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
def upload_images(self):
    """
    Upload the images in this COCO metadata file to GCS.
    The files should be located at the location specified with 
    the `file_name` attribute, either absolute or relative to 
    the location of the COCO file.
    """
    for i, image in self.coco.imgs.items():
        print(image['file_name'])
        # discard the drive letter, if present
        file_name = os.path.splitdrive(image['file_name'])[1]
        splitfile = split_filename(file_name)
        destination = join_urlpath(
            self.relative_gcs_path, *splitfile
        )
        if os.path.isabs(image['file_name']):
            source = image['file_name']
        else:
            source = os.path.join(self.coco_root, image['file_name'])
        self.gcs.upload(self.bucket, source, destination)

dataset_path(datasets_root, organization, project)

Get a url path for uploading/downloading GCS NODD data based on the bucket, organization, etc.

>>> dataset_path('bar', 'pickles+fish', 'project')
'bar/pickles%2Bfish/project'

Parameters:

Name Type Description Default
datasets_root

the root directory within the bucket where we are hosting datasets

 required
organization

the first-level fixed-depth directory for organizing datasets

 required
project

the second-level fixed-depth directory for organizing datasets

 required
Source code in pynoddgcs/publish.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
def dataset_path(datasets_root, organization, project):
    """
    Get a url path for uploading/downloading GCS NODD data based on
    the bucket, organization, etc.

    ```
    >>> dataset_path('bar', 'pickles+fish', 'project')
    'bar/pickles%2Bfish/project'

    ```

    Parameters
    ----------
    datasets_root: str
        the root directory within the bucket where we are hosting datasets
    organization: str
        the first-level fixed-depth directory for organizing datasets
    project: str
        the second-level fixed-depth directory for organizing datasets
    """
    return join_urlpath(
        datasets_root, organization, project)

join_urlpath(*paths)

Joins a bunch of strings into a slash-delimited url path.

>>> join_urlpath('foo', 'bar', 'pickles+fish', 'project')
'foo/bar/pickles%2Bfish/project'

>>> join_urlpath('http://foo', 'bar', 'pickles+fish', 'project')
'http://foo/bar/pickles%2Bfish/project'

>>> join_urlpath('http://foo/bar', 'pickles+fish/project')
'http://foo/bar/pickles%2Bfish/project'

Parameters:

Name Type Description Default
*paths

a variable-length list of path elements to join

 ()

Returns:

Name Type Description
url str

the joined url
Source code in pynoddgcs/publish.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
def join_urlpath(*paths):
    """
    Joins a bunch of strings into a slash-delimited url path.

    ``` 
    >>> join_urlpath('foo', 'bar', 'pickles+fish', 'project')
    'foo/bar/pickles%2Bfish/project'

    >>> join_urlpath('http://foo', 'bar', 'pickles+fish', 'project')
    'http://foo/bar/pickles%2Bfish/project'

    >>> join_urlpath('http://foo/bar', 'pickles+fish/project')
    'http://foo/bar/pickles%2Bfish/project'

    ```

    Parameters
    ----------
    *paths: list[str]
        a variable-length list of path elements to join

    Returns
    -------
    url: str
        the joined url
    """
    url = '/'.join(s.strip('/') for s in paths)
    return urllib.parse.quote(url, safe=':/')

split_filename(filename)

Splits a filename into all of its component directory structure

Does the same thing as os.path.split, but completely splits the directory structure into all parts, instead of just two (head/tail)


>>> split_filename('foo/bar/pickles')
['foo', 'bar', 'pickles']

Parameters:

Name Type Description Default
filename

A filename to split

 required

Returns:

Name Type Description
paths list[str]

The completely split list of directories (and possibly the terminating filename)
Source code in pynoddgcs/publish.py 
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
def split_filename(filename):
    """
    Splits a filename into all of its component directory structure

    Does the same thing as `os.path.split`, but completely splits the 
    directory structure into all parts, instead of just two (head/tail)

    ```

    >>> split_filename('foo/bar/pickles')
    ['foo', 'bar', 'pickles']

    ```

    Parameters
    ----------
    filename: str
        A filename to split

    Returns
    -------
    paths: list[str]
        The completely split list of directories 
        (and possibly the terminating filename)
    """
    paths = []
    tail = "totally_arbitrary"
    while filename and tail:
        filename, tail = os.path.split(filename)
        if tail and tail != '.' and tail != '..':
            paths.append(tail)
    return paths[::-1]