From 2ec714523a071f0b72cf91ce1fe8e412bf97509a Mon Sep 17 00:00:00 2001 From: Timotej S <6674623+underhood@users.noreply.github.com> Date: Thu, 9 Feb 2023 10:18:56 +0700 Subject: generates dual ksy for njfv2 + fix for padding after page blocks (#14383) * add ability to generate different ksys * fix dist * fix page_block by metric count instead of till EOF --- .gitattributes | 1 + .gitignore | 4 + Makefile.am | 15 ++++ database/engine/journalfile.ksy | 144 -------------------------------- database/engine/journalfile_v2.ksy.in | 150 ++++++++++++++++++++++++++++++++++ 5 files changed, 170 insertions(+), 144 deletions(-) delete mode 100644 database/engine/journalfile.ksy create mode 100644 database/engine/journalfile_v2.ksy.in diff --git a/.gitattributes b/.gitattributes index 45ec5156ac..e594b59f86 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ *.c diff=cpp *.h diff=cpp +*.ksy.in linguist-language=ksy diff --git a/.gitignore b/.gitignore index 821d72d912..eeca3b452c 100644 --- a/.gitignore +++ b/.gitignore @@ -231,3 +231,7 @@ Session.*.vim # Judy stuff JudyLTables.c judyltablesgen + +# m4 generated ksys +database/engine/journalfile_v2.ksy +database/engine/journalfile_v2_virtmemb.ksy diff --git a/Makefile.am b/Makefile.am index 3eaf1ee8c2..3ce9cb36fe 100644 --- a/Makefile.am +++ b/Makefile.am @@ -78,6 +78,7 @@ dist_noinst_DATA = \ packaging/protobuf.checksums \ packaging/protobuf.version \ packaging/version \ + database/engine/journalfile_v2.ksy.in \ $(NULL) # until integrated within build @@ -555,6 +556,20 @@ if ENABLE_DBENGINE database/engine/pdc.c \ database/engine/pdc.h \ $(NULL) + + RRD_PLUGIN_KSY_BUILTFILES = \ + database/engine/journalfile_v2.ksy \ + database/engine/journalfile_v2_virtmemb.ksy \ + $(NULL) + + BUILT_SOURCES += $(RRD_PLUGIN_KSY_BUILTFILES) + CLEANFILES += $(RRD_PLUGIN_KSY_BUILTFILES) + +database/engine/journalfile_v2.ksy: $(abs_top_srcdir)/database/engine/journalfile_v2.ksy.in + m4 $(abs_top_srcdir)/database/engine/journalfile_v2.ksy.in > $@ + +database/engine/journalfile_v2_virtmemb.ksy: $(abs_top_srcdir)/database/engine/journalfile_v2.ksy.in + m4 -DVIRT_MEMBERS $(abs_top_srcdir)/database/engine/journalfile_v2.ksy.in > $@ endif API_PLUGIN_FILES = \ diff --git a/database/engine/journalfile.ksy b/database/engine/journalfile.ksy deleted file mode 100644 index 858db83d4b..0000000000 --- a/database/engine/journalfile.ksy +++ /dev/null @@ -1,144 +0,0 @@ -meta: - id: netdata_journalfile_v2 - endian: le - -seq: - - id: journal_v2_header - type: journal_v2_header - size: 4096 - - id: extent_list - type: journal_v2_extent_list - repeat: expr - repeat-expr: journal_v2_header.extent_count - - id: extent_trailer - type: journal_v2_block_trailer - - id: metric_list - type: journal_v2_metric_list - repeat: expr - repeat-expr: journal_v2_header.metric_count - - id: metric_trailer - type: journal_v2_block_trailer - - id: page_blocs - type: jounral_v2_page_blocs - size: _root._io.size - _root._io.pos - 4 - - id: journal_file_trailer - type: journal_v2_block_trailer - - -types: - journal_v2_metric_list: - seq: - - id: uuid - size: 16 - - id: entries - type: u4 - - id: page_offset - type: u4 - - id: delta_start_s - type: u4 - - id: delta_end_s - type: u4 - instances: - page_block: - type: journal_v2_page_block - io: _root._io - pos: page_offset - journal_v2_page_hdr: - seq: - - id: crc - type: u4 - - id: uuid_offset - type: u4 - - id: entries - type: u4 - - id: uuid - size: 16 - journal_v2_page_list: - seq: - - id: delta_start_s - type: u4 - - id: delta_end_s - type: u4 - - id: extent_idx - type: u4 - - id: update_every_s - type: u4 - - id: page_len - type: u2 - - id: type - type: u1 - - id: reserved - type: u1 - instances: - extent: - io: _root._io - type: journal_v2_extent_list - pos: _root.journal_v2_header.extent_offset + (extent_idx * 16) - journal_v2_header: - seq: - - id: magic - contents: [ 0x19, 0x10, 0x22, 0x01 ] #0x01221019 - - id: reserved - type: u4 - - id: start_time_ut - type: u8 - - id: end_time_ut - type: u8 - - id: extent_count - type: u4 - - id: extent_offset - type: u4 - - id: metric_count - type: u4 - - id: metric_offset - type: u4 - - id: page_count - type: u4 - - id: page_offset - type: u4 - - id: extent_trailer_offset - type: u4 - - id: metric_trailer_offset - type: u4 - - id: original_file_size - type: u4 - - id: total_file_size - type: u4 - - id: data - type: u8 - instances: - trailer: - io: _root._io - type: journal_v2_block_trailer - pos: _root._io.size - 4 - journal_v2_block_trailer: - seq: - - id: checksum - type: u4 - journal_v2_extent_list: - seq: - - id: datafile_offset - type: u8 - - id: datafile_size - type: u4 - - id: file_idx - type: u2 - - id: page_cnt - type: u1 - - id: padding - type: u1 - journal_v2_page_block: - seq: - - id: hdr - type: journal_v2_page_hdr - - id: page_list - type: journal_v2_page_list - repeat: expr - repeat-expr: hdr.entries - - id: block_trailer - type: journal_v2_block_trailer - jounral_v2_page_blocs: - seq: - - id: blocs - type: journal_v2_page_block - repeat: eos diff --git a/database/engine/journalfile_v2.ksy.in b/database/engine/journalfile_v2.ksy.in new file mode 100644 index 0000000000..6a656bc454 --- /dev/null +++ b/database/engine/journalfile_v2.ksy.in @@ -0,0 +1,150 @@ +meta: + id: journalfile_v2`'ifdef(`VIRT_MEMBERS',`_virtmemb') + endian: le + application: netdata + file-extension: njfv2 + license: GPL-3.0-or-later + +seq: + - id: journal_v2_header + type: journal_v2_header + size: 4096 + - id: extent_list + type: journal_v2_extent_list + repeat: expr + repeat-expr: journal_v2_header.extent_count + - id: extent_trailer + type: journal_v2_block_trailer + - id: metric_list + type: journal_v2_metric_list + repeat: expr + repeat-expr: journal_v2_header.metric_count + - id: metric_trailer + type: journal_v2_block_trailer + - id: page_blocs + type: journal_v2_page_block + repeat: expr + repeat-expr: _root.journal_v2_header.metric_count + - id: padding + size: _root._io.size - _root._io.pos - 4 + - id: journal_file_trailer + type: journal_v2_block_trailer + +types: + journal_v2_metric_list: + seq: + - id: uuid + size: 16 + - id: entries + type: u4 + - id: page_offset + type: u4 + - id: delta_start_s + type: u4 + - id: delta_end_s + type: u4 +ifdef(`VIRT_MEMBERS', +` instances: + page_block: + type: journal_v2_page_block + io: _root._io + pos: page_offset +')dnl + journal_v2_page_hdr: + seq: + - id: crc + type: u4 + - id: uuid_offset + type: u4 + - id: entries + type: u4 + - id: uuid + size: 16 + journal_v2_page_list: + seq: + - id: delta_start_s + type: u4 + - id: delta_end_s + type: u4 + - id: extent_idx + type: u4 + - id: update_every_s + type: u4 + - id: page_len + type: u2 + - id: type + type: u1 + - id: reserved + type: u1 +ifdef(`VIRT_MEMBERS', +` instances: + extent: + io: _root._io + type: journal_v2_extent_list + pos: _root.journal_v2_header.extent_offset + (extent_idx * 16) +')dnl + journal_v2_header: + seq: + - id: magic + contents: [ 0x19, 0x10, 0x22, 0x01 ] #0x01221019 + - id: reserved + type: u4 + - id: start_time_ut + type: u8 + - id: end_time_ut + type: u8 + - id: extent_count + type: u4 + - id: extent_offset + type: u4 + - id: metric_count + type: u4 + - id: metric_offset + type: u4 + - id: page_count + type: u4 + - id: page_offset + type: u4 + - id: extent_trailer_offset + type: u4 + - id: metric_trailer_offset + type: u4 + - id: original_file_size + type: u4 + - id: total_file_size + type: u4 + - id: data + type: u8 +ifdef(`VIRT_MEMBERS', +` instances: + trailer: + io: _root._io + type: journal_v2_block_trailer + pos: _root._io.size - 4 +')dnl + journal_v2_block_trailer: + seq: + - id: checksum + type: u4 + journal_v2_extent_list: + seq: + - id: datafile_offset + type: u8 + - id: datafile_size + type: u4 + - id: file_idx + type: u2 + - id: page_cnt + type: u1 + - id: padding + type: u1 + journal_v2_page_block: + seq: + - id: hdr + type: journal_v2_page_hdr + - id: page_list + type: journal_v2_page_list + repeat: expr + repeat-expr: hdr.entries + - id: block_trailer + type: journal_v2_block_trailer -- cgit v1.2.3