Skip to content

Commit bbe839f

Browse files
petrelharppetrelharp
committed
padding bytes in json+struct codec
1 parent 2f26dc6 commit bbe839f

6 files changed

Lines changed: 101 additions & 305 deletions

File tree

‎c/tests/test_core.c‎

Lines changed: 0 additions & 156 deletions
Original file line numberDiff line numberDiff line change
@@ -83,161 +83,6 @@ test_generate_uuid(void)
8383
CU_ASSERT_STRING_NOT_EQUAL(uuid, other_uuid);
8484
}
8585

86-
static void
87-
set_u64_le(uint8_t *dest, uint64_t value)
88-
{
89-
dest[0] = (uint8_t) (value & 0xFF);
90-
dest[1] = (uint8_t) ((value >> 8) & 0xFF);
91-
dest[2] = (uint8_t) ((value >> 16) & 0xFF);
92-
dest[3] = (uint8_t) ((value >> 24) & 0xFF);
93-
dest[4] = (uint8_t) ((value >> 32) & 0xFF);
94-
dest[5] = (uint8_t) ((value >> 40) & 0xFF);
95-
dest[6] = (uint8_t) ((value >> 48) & 0xFF);
96-
dest[7] = (uint8_t) ((value >> 56) & 0xFF);
97-
}
98-
99-
static void
100-
test_json_struct_metadata_get_blob(void)
101-
{
102-
int ret;
103-
char metadata[128];
104-
char *json;
105-
tsk_size_t json_buffer_length;
106-
char *blob;
107-
tsk_size_t blob_length;
108-
uint8_t *bytes;
109-
tsk_size_t metadata_length;
110-
size_t header_length;
111-
size_t json_length;
112-
size_t payload_length;
113-
size_t total_length;
114-
char json_payload[] = "{\"a\":1}";
115-
uint8_t binary_payload[] = { 0x01, 0x02, 0x03, 0x04 };
116-
uint8_t empty_payload[] = { 0 };
117-
118-
bytes = (uint8_t *) metadata;
119-
header_length = 4 + 1 + 8 + 8;
120-
json_length = strlen(json_payload);
121-
payload_length = sizeof(binary_payload);
122-
total_length = header_length + json_length + payload_length;
123-
CU_ASSERT_FATAL(total_length <= sizeof(metadata));
124-
memset(metadata, 0, sizeof(metadata));
125-
bytes[0] = 'J';
126-
bytes[1] = 'B';
127-
bytes[2] = 'L';
128-
bytes[3] = 'B';
129-
bytes[4] = 1;
130-
set_u64_le(bytes + 5, (uint64_t) json_length);
131-
set_u64_le(bytes + 13, (uint64_t) payload_length);
132-
memcpy(bytes + header_length, json_payload, json_length);
133-
memcpy(bytes + header_length + json_length, binary_payload, payload_length);
134-
metadata_length = (tsk_size_t) total_length;
135-
ret = tsk_json_struct_metadata_get_blob(
136-
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
137-
CU_ASSERT_EQUAL(ret, 0);
138-
CU_ASSERT_PTR_EQUAL(json, (char *) bytes + header_length);
139-
CU_ASSERT_EQUAL(json_buffer_length, (tsk_size_t) json_length);
140-
if (json_length > 0) {
141-
CU_ASSERT_EQUAL(memcmp(json, json_payload, json_length), 0);
142-
}
143-
CU_ASSERT_PTR_EQUAL(blob, bytes + header_length + json_length);
144-
CU_ASSERT_EQUAL(blob_length, (tsk_size_t) payload_length);
145-
CU_ASSERT_EQUAL(memcmp(blob, binary_payload, payload_length), 0);
146-
147-
payload_length = 0;
148-
total_length = header_length + json_length + payload_length;
149-
CU_ASSERT_FATAL(total_length <= sizeof(metadata));
150-
set_u64_le(bytes + 13, (uint64_t) payload_length);
151-
metadata_length = (tsk_size_t) total_length;
152-
ret = tsk_json_struct_metadata_get_blob(
153-
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
154-
CU_ASSERT_EQUAL(ret, 0);
155-
CU_ASSERT_PTR_EQUAL(json, (char *) bytes + header_length);
156-
CU_ASSERT_EQUAL(json_buffer_length, (tsk_size_t) json_length);
157-
CU_ASSERT_EQUAL(blob_length, (tsk_size_t) payload_length);
158-
CU_ASSERT_PTR_EQUAL(blob, bytes + header_length + json_length);
159-
160-
json_length = 0;
161-
payload_length = sizeof(empty_payload);
162-
total_length = header_length + json_length + payload_length;
163-
CU_ASSERT_FATAL(total_length <= sizeof(metadata));
164-
set_u64_le(bytes + 5, (uint64_t) json_length);
165-
set_u64_le(bytes + 13, (uint64_t) payload_length);
166-
memcpy(bytes + header_length + json_length, empty_payload, payload_length);
167-
metadata_length = (tsk_size_t) total_length;
168-
ret = tsk_json_struct_metadata_get_blob(
169-
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
170-
CU_ASSERT_EQUAL(ret, 0);
171-
CU_ASSERT_PTR_EQUAL(json, (char *) bytes + header_length);
172-
CU_ASSERT_EQUAL(json_buffer_length, (tsk_size_t) json_length);
173-
CU_ASSERT_EQUAL(blob_length, (tsk_size_t) payload_length);
174-
CU_ASSERT_PTR_EQUAL(blob, bytes + header_length + json_length);
175-
CU_ASSERT_EQUAL(memcmp(blob, empty_payload, payload_length), 0);
176-
177-
blob = NULL;
178-
blob_length = 0;
179-
json = NULL;
180-
json_buffer_length = 0;
181-
metadata_length = header_length - 1;
182-
ret = tsk_json_struct_metadata_get_blob(
183-
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
184-
CU_ASSERT_EQUAL(ret, TSK_ERR_JSON_STRUCT_METADATA_TRUNCATED);
185-
186-
metadata_length = (tsk_size_t) total_length;
187-
bytes[0] = 'X';
188-
ret = tsk_json_struct_metadata_get_blob(
189-
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
190-
CU_ASSERT_EQUAL(ret, TSK_ERR_JSON_STRUCT_METADATA_BAD_MAGIC);
191-
bytes[0] = 'J';
192-
193-
bytes[4] = 2;
194-
ret = tsk_json_struct_metadata_get_blob(
195-
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
196-
CU_ASSERT_EQUAL(ret, TSK_ERR_JSON_STRUCT_METADATA_BAD_VERSION);
197-
bytes[4] = 1;
198-
199-
metadata_length = (tsk_size_t) (total_length - 1);
200-
ret = tsk_json_struct_metadata_get_blob(
201-
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
202-
CU_ASSERT_EQUAL(ret, TSK_ERR_JSON_STRUCT_METADATA_TRUNCATED);
203-
204-
ret = tsk_json_struct_metadata_get_blob(
205-
NULL, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
206-
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
207-
ret = tsk_json_struct_metadata_get_blob(
208-
metadata, metadata_length, NULL, &json_buffer_length, &blob, &blob_length);
209-
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
210-
ret = tsk_json_struct_metadata_get_blob(
211-
metadata, metadata_length, &json, NULL, &blob, &blob_length);
212-
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
213-
ret = tsk_json_struct_metadata_get_blob(
214-
metadata, metadata_length, &json, &json_buffer_length, NULL, &blob_length);
215-
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
216-
ret = tsk_json_struct_metadata_get_blob(
217-
metadata, metadata_length, &json, &json_buffer_length, &blob, NULL);
218-
CU_ASSERT_EQUAL(ret, TSK_ERR_BAD_PARAM_VALUE);
219-
220-
memset(metadata, 0, sizeof(metadata));
221-
bytes[0] = 'J';
222-
bytes[1] = 'B';
223-
bytes[2] = 'L';
224-
bytes[3] = 'B';
225-
bytes[4] = 1;
226-
metadata_length = (tsk_size_t) header_length;
227-
228-
set_u64_le(bytes + 5, UINT64_MAX - (uint64_t) header_length + 1);
229-
set_u64_le(bytes + 13, 0);
230-
ret = tsk_json_struct_metadata_get_blob(
231-
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
232-
CU_ASSERT_EQUAL(ret, TSK_ERR_JSON_STRUCT_METADATA_INVALID_LENGTH);
233-
234-
set_u64_le(bytes + 5, 8);
235-
set_u64_le(bytes + 13, UINT64_MAX - (uint64_t) (header_length + 8) + 1);
236-
ret = tsk_json_struct_metadata_get_blob(
237-
metadata, metadata_length, &json, &json_buffer_length, &blob, &blob_length);
238-
CU_ASSERT_EQUAL(ret, TSK_ERR_JSON_STRUCT_METADATA_INVALID_LENGTH);
239-
}
240-
24186
static void
24287
test_double_round(void)
24388
{
@@ -808,7 +653,6 @@ main(int argc, char **argv)
808653
{ "test_strerror", test_strerror },
809654
{ "test_strerror_kastore", test_strerror_kastore },
810655
{ "test_generate_uuid", test_generate_uuid },
811-
{ "test_json_struct_metadata_get_blob", test_json_struct_metadata_get_blob },
812656
{ "test_double_round", test_double_round },
813657
{ "test_blkalloc", test_blkalloc },
814658
{ "test_unknown_time", test_unknown_time },

‎c/tskit/core.c‎

Lines changed: 1 addition & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,7 @@
3232
#include <kastore.h>
3333
#include <tskit/core.h>
3434

35-
#define UUID_NUM_BYTES 16
36-
#define TSK_JSON_BINARY_HEADER_SIZE 21
37-
38-
static const uint8_t _tsk_json_binary_magic[4] = { 'J', 'B', 'L', 'B' };
35+
#define UUID_NUM_BYTES 16
3936

4037
#if defined(_WIN32)
4138

@@ -98,22 +95,6 @@ get_random_bytes(uint8_t *buf)
9895

9996
#endif
10097

101-
static uint64_t
102-
tsk_load_u64_le(const uint8_t *p)
103-
{
104-
uint64_t value;
105-
106-
value = (uint64_t) p[0];
107-
value |= (uint64_t) p[1] << 8;
108-
value |= (uint64_t) p[2] << 16;
109-
value |= (uint64_t) p[3] << 24;
110-
value |= (uint64_t) p[4] << 32;
111-
value |= (uint64_t) p[5] << 40;
112-
value |= (uint64_t) p[6] << 48;
113-
value |= (uint64_t) p[7] << 56;
114-
return value;
115-
}
116-
11798
/* Generate a new UUID4 using a system-generated source of randomness.
11899
* Note that this function writes a NULL terminator to the end of this
119100
* string, so that the total length of the buffer must be 37 bytes.
@@ -141,65 +122,6 @@ tsk_generate_uuid(char *dest, int TSK_UNUSED(flags))
141122
return ret;
142123
}
143124

144-
int
145-
tsk_json_struct_metadata_get_blob(char *metadata, tsk_size_t metadata_length,
146-
char **json, tsk_size_t *json_length, char **blob, tsk_size_t *blob_length)
147-
{
148-
int ret;
149-
uint8_t version;
150-
uint64_t json_length_u64;
151-
uint64_t binary_length_u64;
152-
uint64_t header_and_json_length;
153-
uint64_t total_length;
154-
uint8_t *bytes;
155-
char *blob_start;
156-
char *json_start;
157-
158-
if (metadata == NULL || json == NULL || json_length == NULL || blob == NULL
159-
|| blob_length == NULL) {
160-
ret = tsk_trace_error(TSK_ERR_BAD_PARAM_VALUE);
161-
goto out;
162-
}
163-
bytes = (uint8_t *) metadata;
164-
if (metadata_length < TSK_JSON_BINARY_HEADER_SIZE) {
165-
ret = tsk_trace_error(TSK_ERR_JSON_STRUCT_METADATA_TRUNCATED);
166-
goto out;
167-
}
168-
if (memcmp(bytes, _tsk_json_binary_magic, sizeof(_tsk_json_binary_magic)) != 0) {
169-
ret = tsk_trace_error(TSK_ERR_JSON_STRUCT_METADATA_BAD_MAGIC);
170-
goto out;
171-
}
172-
version = bytes[4];
173-
if (version != 1) {
174-
ret = tsk_trace_error(TSK_ERR_JSON_STRUCT_METADATA_BAD_VERSION);
175-
goto out;
176-
}
177-
json_length_u64 = tsk_load_u64_le(bytes + 5);
178-
binary_length_u64 = tsk_load_u64_le(bytes + 13);
179-
if (json_length_u64 > UINT64_MAX - (uint64_t) TSK_JSON_BINARY_HEADER_SIZE) {
180-
ret = tsk_trace_error(TSK_ERR_JSON_STRUCT_METADATA_INVALID_LENGTH);
181-
goto out;
182-
}
183-
header_and_json_length = (uint64_t) TSK_JSON_BINARY_HEADER_SIZE + json_length_u64;
184-
if (binary_length_u64 > UINT64_MAX - header_and_json_length) {
185-
ret = tsk_trace_error(TSK_ERR_JSON_STRUCT_METADATA_INVALID_LENGTH);
186-
goto out;
187-
}
188-
total_length = header_and_json_length + binary_length_u64;
189-
if ((uint64_t) metadata_length < total_length) {
190-
ret = tsk_trace_error(TSK_ERR_JSON_STRUCT_METADATA_TRUNCATED);
191-
goto out;
192-
}
193-
json_start = (char *) bytes + TSK_JSON_BINARY_HEADER_SIZE;
194-
blob_start = (char *) bytes + TSK_JSON_BINARY_HEADER_SIZE + json_length_u64;
195-
*json = json_start;
196-
*json_length = (tsk_size_t) json_length_u64;
197-
*blob = blob_start;
198-
*blob_length = (tsk_size_t) binary_length_u64;
199-
ret = 0;
200-
out:
201-
return ret;
202-
}
203125
static const char *
204126
tsk_strerror_internal(int err)
205127
{
@@ -267,22 +189,6 @@ tsk_strerror_internal(int err)
267189
ret = "An incompatible type for a column was found in the file. "
268190
"(TSK_ERR_BAD_COLUMN_TYPE)";
269191
break;
270-
case TSK_ERR_JSON_STRUCT_METADATA_BAD_MAGIC:
271-
ret = "JSON binary struct metadata does not begin with the expected "
272-
"magic bytes. (TSK_ERR_JSON_STRUCT_METADATA_BAD_MAGIC)";
273-
break;
274-
case TSK_ERR_JSON_STRUCT_METADATA_TRUNCATED:
275-
ret = "JSON binary struct metadata is shorter than the expected size. "
276-
"(TSK_ERR_JSON_STRUCT_METADATA_TRUNCATED)";
277-
break;
278-
case TSK_ERR_JSON_STRUCT_METADATA_INVALID_LENGTH:
279-
ret = "A length field in the JSON binary struct metadata header is invalid. "
280-
"(TSK_ERR_JSON_STRUCT_METADATA_INVALID_LENGTH)";
281-
break;
282-
case TSK_ERR_JSON_STRUCT_METADATA_BAD_VERSION:
283-
ret = "JSON binary struct metadata uses an unsupported version number. "
284-
"(TSK_ERR_JSON_STRUCT_METADATA_BAD_VERSION)";
285-
break;
286192

287193
/* Out of bounds errors */
288194
case TSK_ERR_BAD_OFFSET:

‎c/tskit/core.h‎

Lines changed: 0 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -309,26 +309,6 @@ not found in the file.
309309
An unsupported type was provided for a column in the file.
310310
*/
311311
#define TSK_ERR_BAD_COLUMN_TYPE -105
312-
313-
/**
314-
The JSON binary struct metadata does not begin with the expected magic bytes.
315-
*/
316-
#define TSK_ERR_JSON_STRUCT_METADATA_BAD_MAGIC -106
317-
318-
/**
319-
The JSON binary struct metadata is shorter than the expected size.
320-
*/
321-
#define TSK_ERR_JSON_STRUCT_METADATA_TRUNCATED -107
322-
323-
/**
324-
A length field in the JSON binary struct metadata header is invalid.
325-
*/
326-
#define TSK_ERR_JSON_STRUCT_METADATA_INVALID_LENGTH -108
327-
328-
/**
329-
The JSON binary struct metadata uses an unsupported version number.
330-
*/
331-
#define TSK_ERR_JSON_STRUCT_METADATA_BAD_VERSION -109
332312
/** @} */
333313

334314
/**
@@ -1132,30 +1112,6 @@ bool tsk_isfinite(double val);
11321112
#define TSK_UUID_SIZE 36
11331113
int tsk_generate_uuid(char *dest, int flags);
11341114

1135-
/**
1136-
@brief Extract the binary payload from ``json+struct`` encoded metadata.
1137-
1138-
@rst
1139-
Metadata produced by the JSONStructCodec consists of a fixed-size
1140-
header followed by canonical JSON bytes and an optional binary payload. This helper
1141-
validates the framing, returning pointers to the embedded JSON and binary sections
1142-
without copying.
1143-
1144-
The output pointers reference memory owned by the caller and remain valid only while
1145-
the original metadata buffer is alive.
1146-
@endrst
1147-
1148-
@param[in] metadata Pointer to the encoded metadata bytes.
1149-
@param[in] metadata_length Number of bytes available at ``metadata``.
1150-
@param[out] json On success, set to the start of the JSON bytes.
1151-
@param[out] json_length On success, set to the JSON length in bytes.
1152-
@param[out] blob On success, set to the start of the binary payload.
1153-
@param[out] blob_length On success, set to the payload length in bytes.
1154-
@return Return 0 on success or a negative value on failure.
1155-
*/
1156-
int tsk_json_struct_metadata_get_blob(char *metadata, tsk_size_t metadata_length,
1157-
char **json, tsk_size_t *json_length, char **blob, tsk_size_t *blob_length);
1158-
11591115
/* TODO most of these can probably be macros so they compile out as no-ops.
11601116
* Lets do the 64 bit tsk_size_t switch first though. */
11611117
void *tsk_malloc(tsk_size_t size);

‎docs/c-api.rst‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -620,8 +620,6 @@ Miscellaneous functions
620620

621621
.. doxygenfunction:: tsk_is_unknown_time
622622

623-
.. doxygenfunction:: tsk_json_struct_metadata_get_blob
624-
625623
*************************
626624
Function Specific Options
627625
*************************

0 commit comments

Comments
 (0)