Encoding and decoding files

In this example we show how to encode/decode files with Kodo. In Kodo this is example is nearly identical to the example Encoding and decoding large objects. For this reason we will mainly highlight the differences.

The complete example

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
// Copyright Steinwurf ApS 2011.
// Distributed under the "STEINWURF RESEARCH LICENSE 1.0".
// See accompanying file LICENSE.rst or
// http://www.steinwurf.com/licensing


//! [0]
#include <kodo_core/object/file_encoder.hpp>
#include <kodo_core/object/file_decoder.hpp>

#include <kodo_rlnc/full_vector_codes.hpp>
//! [1]

#include <cassert>
#include <cstdint>
#include <fstream>
#include <iostream>
#include <vector>

/// @example encode_decode_file.cpp
///
/// Often we want to encode / decode data that exceed a single
/// encoding/decoding block. In this case we need to "chop" up
/// the data into manageable chunks and then encode and decode
/// each chuck separately. This example shows how to use the
/// file encoder in Kodo. The file encoder operates directly on
/// a file in the file-system. For decoding we use an object decoder
/// which decodes data to memory, but which is compatible with
/// file encoder.

int main()
{
    //! [2]
    // Set the number of symbols (i.e. the generation size in RLNC
    // terminology) and the size of a symbol in bytes
    uint32_t symbols = 42;
    uint32_t symbol_size = 64;
    fifi::api::field field = fifi::api::field::binary;

    uint32_t file_size = 23456;

    std::string encode_filename = "encode-file.bin";
    std::string decode_filename = "decode-file.bin";

    using file_encoder = kodo_core::object::file_encoder<
                         kodo_rlnc::full_vector_encoder>;

    using file_decoder = kodo_core::object::file_decoder<
                         kodo_rlnc::full_vector_decoder>;
    //! [3]

    //! [4]
    // Create a test file for encoding.
    std::ofstream encode_file;

    encode_file.open(encode_filename, std::ios::binary);
    std::vector<char> data_in(file_size, 'x');

    encode_file.write(data_in.data(), data_in.size());
    encode_file.close();
    //! [5]

    //! [6]
    // Actual encoding/decoding of the file
    file_encoder::factory encoder_factory(field, symbols, symbol_size);
    file_decoder::factory decoder_factory(field, symbols, symbol_size);

    encoder_factory.set_filename(encode_filename);

    decoder_factory.set_filename(decode_filename);
    decoder_factory.set_file_size(file_size);

    auto encoder = encoder_factory.build();
    auto decoder = decoder_factory.build();

    std::cout << "encoder blocks = " << encoder->blocks() << std::endl;
    std::cout << "decoder blocks = " << decoder->blocks() << std::endl;
    //! [7]

    //! [8]
    for (uint32_t i = 0; i < encoder->blocks(); ++i)
    {
        file_encoder::stack_pointer e = encoder->build(i);
        file_decoder::stack_pointer d = decoder->build(i);
        std::vector<uint8_t> payload(e->payload_size());

        while (!d->is_complete())
        {
            // Comment in the following function to turn systematic off
            // e->set_systematic_off();
            e->write_payload(payload.data());

            // Here we would send and receive the payload over a
            // network. Lets throw away some packet to simulate.
            if (rand() % 2)
            {
                continue;
            }

            d->read_payload(payload.data());
        }
    }
    //! [9]
}

Adding the includes

First we have to provide the appropriate includes which defines the codec that we want to use and the kodo::object::file_encoder and kodo::object::file_decoder classes.

1
2
3
4
#include <kodo_core/object/file_encoder.hpp>
#include <kodo_core/object/file_decoder.hpp>

#include <kodo_rlnc/full_vector_codes.hpp>

Specifying the coding parameters

For the file encoder/decoder case three options are new. The first is the file name of the file we want to encode, the seconds is the file name of the file we want to decode data into and finally the size of file.

Note

In a real application we would most likely not use different file names for the encoder and decoder.

Note

The file size is only needed by the file decoder. The file encoder knows the file size after opening the file.

As with the Encoding and decoding large objects we pass type of the actual encoding and decoding algorithm as a template argument.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
    // Set the number of symbols (i.e. the generation size in RLNC
    // terminology) and the size of a symbol in bytes
    uint32_t symbols = 42;
    uint32_t symbol_size = 64;
    fifi::api::field field = fifi::api::field::binary;

    uint32_t file_size = 23456;

    std::string encode_filename = "encode-file.bin";
    std::string decode_filename = "decode-file.bin";

    using file_encoder = kodo_core::object::file_encoder<
                         kodo_rlnc::full_vector_encoder>;

    using file_decoder = kodo_core::object::file_decoder<
                         kodo_rlnc::full_vector_decoder>;

Creating a test file

Here we create a test file to use for the encoder. This is just for the sake of the example.

1
2
3
4
5
6
7
8
    // Create a test file for encoding.
    std::ofstream encode_file;

    encode_file.open(encode_filename, std::ios::binary);
    std::vector<char> data_in(file_size, 'x');

    encode_file.write(data_in.data(), data_in.size());
    encode_file.close();

Using the file encoder and decoder

As with the storage encoders we now build the file encoder and decoder.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
    storage_encoder::factory encoder_factory(
        field, symbols, symbol_size);
    storage_decoder::factory decoder_factory(
        field, symbols, symbol_size);

    auto encoder = encoder_factory.build();
    auto decoder = decoder_factory.build();

    std::vector<uint8_t> data_in(object_size, 'x');
    std::vector<uint8_t> data_out(object_size, '\0');

    encoder->set_const_storage(storage::storage(data_in));
    decoder->set_mutable_storage(storage::storage(data_out));

    assert(encoder->object_size() == object_size);
    assert(decoder->object_size() == object_size);

    std::cout << "object_size = " << object_size << std::endl;
    std::cout << "encoder blocks = " << encoder->blocks() << std::endl;
    std::cout << "decoder blocks = " << decoder->blocks() << std::endl;

Also the encoding/decoding loop is similar to the Encoding and decoding large objects example since we potentially need more than one encoder/decoder pair to code the entire file.

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
    for (uint32_t i = 0; i < encoder->blocks(); ++i)
    {
        storage_encoder::stack_pointer e = encoder->build(i);
        storage_decoder::stack_pointer d = decoder->build(i);

        std::vector<uint8_t> payload(e->payload_size());

        while (!d->is_complete())
        {
            e->write_payload(payload.data());

            // Here we would send and receive the payload over a
            // network. Lets throw away some packet to simulate.
            if (rand() % 2)
            {
                continue;
            }

            d->read_payload(payload.data());
        }
    }

    // Check we properly decoded the data
    if (data_in == data_out)
    {
        std::cout << "Data decoded correctly" << std::endl;
    }
    else
    {
        std::cout << "Unexpected failure to decode "
                  << "please file a bug report :)" << std::endl;
    }