Skip to content

Commit ca2a6a5

Browse files
edsiperedsiper
committed
pack: add metadata and group info into log JSON record (fix #10258)
Before this patch, the JSON log record encoder only supported the log record body as an output content, missing support for log record metadata and group metadata and group attributes (if defined). This patch makes the JSON encoder add support for the missing data, packaging into the record a new parent key called '__internal__' that holds the following inside it: - group_attributes: group header attributes - log_metadata: log record metadata Note that for cases where the origin of the data was OTLP and as part of the groups we add 'group metadata' (in the origin), this is not being used since it only contains internal references that are not used for this JSON export: we do not aim to reassmble this JSON into a Fluent Bit record. Consider the following original OTLP JSON record ingested into the pipeline: { "resourceLogs": [ { "resource": { "attributes": [ { "key": "service.name", "value": { "stringValue": "my.service" } } ] }, "scopeLogs": [ { "scope": { "name": "my.library", "version": "1.0.0", "attributes": [ { "key": "my.scope.attribute", "value": { "stringValue": "some scope attribute" } } ] }, "logRecords": [ { "timeUnixNano": "1544712660300000000", "observedTimeUnixNano": "1544712660300000000", "severityNumber": 10, "severityText": "Information", "traceId": "5B8EFFF798038103D269B633813FC60C", "spanId": "EEE19B7EC3C1B174", "body": { "stringValue": "Example log record" }, "attributes": [ { "key": "string.attribute", "value": { "stringValue": "some string" } }, { "key": "boolean.attribute", "value": { "boolValue": true } }, { "key": "int.attribute", "value": { "intValue": "10" } }, { "key": "double.attribute", "value": { "doubleValue": 637.704 } }, { "key": "array.attribute", "value": { "arrayValue": { "values": [ { "stringValue": "many" }, { "stringValue": "values" } ] } } }, { "key": "map.attribute", "value": { "kvlistValue": { "values": [ { "key": "some.map.key", "value": { "stringValue": "some value" } } ] } } } ] } ], "schemaUrl": "https://example.com/schema" } ], "schemaUrl": "https://example.com/schema" } ] } now when encoded in msgpack and then JSON the output will be: { "date": 1544712658.274599, "__internal__": { "group_attributes": { "resource": { "attributes": { "service.name": "my.service" } }, "scope": { "name": "my.library", "version": "1.0.0", "attributes": { "my.scope.attribute": "some scope attribute" } } }, "log_metadata": { "otlp": { "observed_timestamp": 1544712660300000000, "severity_number": 10, "severity_text": "Information", "attributes": { "string.attribute": "some string", "boolean.attribute": true, "int.attribute": 10, "double.attribute": 637.704, "array.attribute": [ "many", "values" ], "map.attribute": { "some.map.key": "some value" } }, "trace_id": "5B8EFFF798038103D269B633813FC60C", "span_id": "EEE19B7EC3C1B174" } } }, "log": "Example log record" } Signed-off-by: Eduardo Silva <eduardo@chronosphere.io>
1 parent 757b13d commit ca2a6a5

File tree

1 file changed

+113
-52
lines changed

1 file changed

+113
-52
lines changed

‎src/flb_pack.c

Lines changed: 113 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@
3030
#include <fluent-bit/flb_pack.h>
3131
#include <fluent-bit/flb_unescape.h>
3232

33+
#include <fluent-bit/flb_log_event_encoder.h>
34+
#include <fluent-bit/flb_log_event_decoder.h>
35+
3336
/* cmetrics */
3437
#include <cmetrics/cmetrics.h>
3538
#include <cmetrics/cmt_decode_msgpack.h>
@@ -917,23 +920,21 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
917920
flb_sds_t date_key)
918921
{
919922
int i;
920-
int ok = MSGPACK_UNPACK_SUCCESS;
921-
int records = 0;
922-
int map_size;
923-
size_t off = 0;
923+
int ret;
924924
char time_formatted[38];
925925
flb_sds_t out_tmp;
926926
flb_sds_t out_js;
927927
flb_sds_t out_buf = NULL;
928-
msgpack_unpacked result;
929-
msgpack_object root;
930-
msgpack_object map;
931928
msgpack_sbuffer tmp_sbuf;
932929
msgpack_packer tmp_pck;
933-
msgpack_object *obj;
934930
msgpack_object *k;
935931
msgpack_object *v;
936932
struct flb_time tms;
933+
struct flb_log_event_decoder log_decoder;
934+
struct flb_log_event log_event;
935+
struct flb_mp_map_header mh_array;
936+
struct flb_mp_map_header mh_map;
937+
struct flb_mp_map_header mh_internal;
937938

938939
/* For json lines and streams mode we need a pre-allocated buffer */
939940
if (json_format == FLB_PACK_JSON_FORMAT_LINES ||
@@ -949,6 +950,15 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
949950
msgpack_sbuffer_init(&tmp_sbuf);
950951
msgpack_packer_init(&tmp_pck, &tmp_sbuf, msgpack_sbuffer_write);
951952

953+
ret = flb_log_event_decoder_init(&log_decoder, (char *) data, bytes);
954+
if (ret != FLB_EVENT_DECODER_SUCCESS) {
955+
flb_error("Log event decoder initialization error : %d", ret);
956+
if (out_buf) {
957+
flb_sds_destroy(out_buf);
958+
}
959+
return NULL;
960+
}
961+
952962
/*
953963
* If the format is the original msgpack style of one big array,
954964
* registrate the array, otherwise is not necessary. FYI, original format:
@@ -960,43 +970,25 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
960970
* ]
961971
*/
962972
if (json_format == FLB_PACK_JSON_FORMAT_JSON) {
963-
records = flb_mp_count(data, bytes);
964-
if (records <= 0) {
965-
msgpack_sbuffer_destroy(&tmp_sbuf);
966-
return NULL;
967-
}
968-
msgpack_pack_array(&tmp_pck, records);
973+
/* register the array. Note must be finalized with flb_mp_map_header_end() */
974+
flb_mp_array_header_init(&mh_array, &tmp_pck);
969975
}
970976

971-
msgpack_unpacked_init(&result);
972-
while (msgpack_unpack_next(&result, data, bytes, &off) == ok) {
973-
/* Each array must have two entries: time and record */
974-
root = result.data;
975-
if (root.type != MSGPACK_OBJECT_ARRAY) {
976-
continue;
977-
}
978-
if (root.via.array.size != 2) {
979-
continue;
977+
/* Iterate log records */
978+
while ((ret = flb_log_event_decoder_next(&log_decoder, &log_event)) == FLB_EVENT_DECODER_SUCCESS) {
979+
if (json_format == FLB_PACK_JSON_FORMAT_JSON) {
980+
/* register a new entry for the array entry */
981+
flb_mp_array_header_append(&mh_array);
980982
}
983+
tms = log_event.timestamp;
981984

982-
/* Unpack time */
983-
flb_time_pop_from_msgpack(&tms, &result, &obj);
984-
985-
/* Get the record/map */
986-
map = root.via.array.ptr[1];
987-
if (map.type != MSGPACK_OBJECT_MAP) {
988-
continue;
989-
}
990-
map_size = map.via.map.size;
985+
/* initialize the map for the record key/values */
986+
flb_mp_map_header_init(&mh_map, &tmp_pck);
991987

988+
/* date key */
992989
if (date_key != NULL) {
993-
msgpack_pack_map(&tmp_pck, map_size + 1);
994-
}
995-
else {
996-
msgpack_pack_map(&tmp_pck, map_size);
997-
}
990+
flb_mp_array_header_append(&mh_map);
998991

999-
if (date_key != NULL) {
1000992
/* Append date key */
1001993
msgpack_pack_str(&tmp_pck, flb_sds_len(date_key));
1002994
msgpack_pack_str_body(&tmp_pck, date_key, flb_sds_len(date_key));
@@ -1011,7 +1003,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
10111003
FLB_PACK_JSON_DATE_JAVA_SQL_TIMESTAMP_FMT, ".%06" PRIu64)) {
10121004
flb_sds_destroy(out_buf);
10131005
msgpack_sbuffer_destroy(&tmp_sbuf);
1014-
msgpack_unpacked_destroy(&result);
1006+
flb_log_event_decoder_destroy(&log_decoder);
10151007
return NULL;
10161008
}
10171009
break;
@@ -1020,7 +1012,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
10201012
FLB_PACK_JSON_DATE_ISO8601_FMT, ".%06" PRIu64 "Z")) {
10211013
flb_sds_destroy(out_buf);
10221014
msgpack_sbuffer_destroy(&tmp_sbuf);
1023-
msgpack_unpacked_destroy(&result);
1015+
flb_log_event_decoder_destroy(&log_decoder);
10241016
return NULL;
10251017
}
10261018
break;
@@ -1033,12 +1025,77 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
10331025
}
10341026
}
10351027

1036-
/* Append remaining keys/values */
1037-
for (i = 0; i < map_size; i++) {
1038-
k = &map.via.map.ptr[i].key;
1039-
v = &map.via.map.ptr[i].val;
1040-
msgpack_pack_object(&tmp_pck, *k);
1041-
msgpack_pack_object(&tmp_pck, *v);
1028+
/* register __internal__ data that comes from the group information */
1029+
if ((log_event.group_attributes && log_event.group_attributes->type == MSGPACK_OBJECT_MAP && log_event.group_attributes->via.map.size > 0) ||
1030+
(log_event.metadata && log_event.metadata->type == MSGPACK_OBJECT_MAP && log_event.metadata->via.map.size > 0)) {
1031+
1032+
flb_mp_map_header_append(&mh_map);
1033+
msgpack_pack_str(&tmp_pck, 12);
1034+
msgpack_pack_str_body(&tmp_pck, "__internal__", 12);
1035+
1036+
flb_mp_map_header_init(&mh_internal, &tmp_pck);
1037+
1038+
/*
1039+
* group metadata: the JSON export of this record do not aim to be re-assembled into a Fluent pipeline,
1040+
* actually it is a generic JSON representation of the log record. For this reason, we need to add the group
1041+
* metadata to the JSON output.
1042+
*
1043+
* Just leaving this code commented as a reference...
1044+
*/
1045+
1046+
/*
1047+
* if (log_event.group_metadata != NULL) {
1048+
* flb_mp_map_header_append(&mh_internal);
1049+
* msgpack_pack_str(&tmp_pck, 14);
1050+
* msgpack_pack_str_body(&tmp_pck, "group_metadata", 14);
1051+
* msgpack_pack_object(&tmp_pck, *log_event.group_metadata);
1052+
* }
1053+
*/
1054+
1055+
/* Append group attributes */
1056+
if (log_event.group_attributes != NULL) {
1057+
flb_mp_map_header_append(&mh_internal);
1058+
msgpack_pack_str(&tmp_pck, 16);
1059+
msgpack_pack_str_body(&tmp_pck, "group_attributes", 16);
1060+
msgpack_pack_object(&tmp_pck, *log_event.group_attributes);
1061+
}
1062+
1063+
/* log/record metadata if exists */
1064+
if (log_event.metadata != NULL) {
1065+
flb_mp_map_header_append(&mh_internal);
1066+
msgpack_pack_str(&tmp_pck, 12);
1067+
msgpack_pack_str_body(&tmp_pck, "log_metadata", 12);
1068+
msgpack_pack_object(&tmp_pck, *log_event.metadata);
1069+
}
1070+
1071+
/* finalize the internal map */
1072+
flb_mp_map_header_end(&mh_internal);
1073+
}
1074+
1075+
/* Append keys/values from the log body */
1076+
if (log_event.body != NULL) {
1077+
if (log_event.body->type == MSGPACK_OBJECT_MAP) {
1078+
for (i = 0; i < log_event.body->via.map.size; i++) {
1079+
flb_mp_map_header_append(&mh_map);
1080+
k = &log_event.body->via.map.ptr[i].key;
1081+
v = &log_event.body->via.map.ptr[i].val;
1082+
1083+
/* Append key/value */
1084+
msgpack_pack_object(&tmp_pck, *k);
1085+
msgpack_pack_object(&tmp_pck, *v);
1086+
}
1087+
1088+
flb_mp_map_header_end(&mh_map);
1089+
}
1090+
else {
1091+
/* for any other data type, nest the content inside log */
1092+
flb_mp_map_header_append(&mh_map);
1093+
msgpack_pack_str(&tmp_pck, 4);
1094+
msgpack_pack_str_body(&tmp_pck, "log", 3);
1095+
msgpack_pack_object(&tmp_pck, *log_event.body);
1096+
1097+
flb_mp_map_header_end(&mh_map);
1098+
}
10421099
}
10431100

10441101
/*
@@ -1071,7 +1128,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
10711128
if (!out_js) {
10721129
flb_sds_destroy(out_buf);
10731130
msgpack_sbuffer_destroy(&tmp_sbuf);
1074-
msgpack_unpacked_destroy(&result);
1131+
flb_log_event_decoder_destroy(&log_decoder);
10751132
return NULL;
10761133
}
10771134

@@ -1084,7 +1141,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
10841141
flb_sds_destroy(out_js);
10851142
flb_sds_destroy(out_buf);
10861143
msgpack_sbuffer_destroy(&tmp_sbuf);
1087-
msgpack_unpacked_destroy(&result);
1144+
flb_log_event_decoder_destroy(&log_decoder);
10881145
return NULL;
10891146
}
10901147

@@ -1102,7 +1159,7 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
11021159
if (!out_tmp) {
11031160
flb_sds_destroy(out_buf);
11041161
msgpack_sbuffer_destroy(&tmp_sbuf);
1105-
msgpack_unpacked_destroy(&result);
1162+
flb_log_event_decoder_destroy(&log_decoder);
11061163
return NULL;
11071164
}
11081165
if (out_tmp != out_buf) {
@@ -1113,14 +1170,18 @@ flb_sds_t flb_pack_msgpack_to_json_format(const char *data, uint64_t bytes,
11131170
}
11141171
}
11151172

1116-
/* Release the unpacker */
1117-
msgpack_unpacked_destroy(&result);
1173+
/* destroy the decoder */
1174+
flb_log_event_decoder_destroy(&log_decoder);
1175+
1176+
/* finalize the main array */
1177+
if (json_format == FLB_PACK_JSON_FORMAT_JSON) {
1178+
flb_mp_array_header_end(&mh_array);
1179+
}
11181180

11191181
/* Format to JSON */
11201182
if (json_format == FLB_PACK_JSON_FORMAT_JSON) {
11211183
out_buf = flb_msgpack_raw_to_json_sds(tmp_sbuf.data, tmp_sbuf.size);
11221184
msgpack_sbuffer_destroy(&tmp_sbuf);
1123-
11241185
if (!out_buf) {
11251186
return NULL;
11261187
}

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy