generates dual ksy for njfv2 + fix for padding after page blocks (#14383)

* add ability to generate different ksys

* fix dist

* fix page_block by metric count instead of till EOF

4 files changed, 37 insertions, 11 deletions

diff --git a/.gitattributes b/.gitattributes
index 45ec5156ac..e594b59f86 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -1,2 +1,3 @@
 *.c	diff=cpp
 *.h	diff=cpp
+*.ksy.in linguist-language=ksy
diff --git a/.gitignore b/.gitignore
index 821d72d912..eeca3b452c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -231,3 +231,7 @@ Session.*.vim
 # Judy stuff
 JudyLTables.c
 judyltablesgen
+
+# m4 generated ksys
+database/engine/journalfile_v2.ksy
+database/engine/journalfile_v2_virtmemb.ksy
diff --git a/Makefile.am b/Makefile.am
index 3eaf1ee8c2..3ce9cb36fe 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -78,6 +78,7 @@ dist_noinst_DATA = \
     packaging/protobuf.checksums \
     packaging/protobuf.version \
     packaging/version \
+    database/engine/journalfile_v2.ksy.in \
     $(NULL)
 
 # until integrated within build
@@ -555,6 +556,20 @@ if ENABLE_DBENGINE
         database/engine/pdc.c \
         database/engine/pdc.h \
         $(NULL)
+    
+    RRD_PLUGIN_KSY_BUILTFILES = \
+        database/engine/journalfile_v2.ksy \
+        database/engine/journalfile_v2_virtmemb.ksy \
+        $(NULL)
+
+    BUILT_SOURCES += $(RRD_PLUGIN_KSY_BUILTFILES)
+    CLEANFILES += $(RRD_PLUGIN_KSY_BUILTFILES)
+
+database/engine/journalfile_v2.ksy: $(abs_top_srcdir)/database/engine/journalfile_v2.ksy.in
+	m4 $(abs_top_srcdir)/database/engine/journalfile_v2.ksy.in > $@
+
+database/engine/journalfile_v2_virtmemb.ksy: $(abs_top_srcdir)/database/engine/journalfile_v2.ksy.in
+	m4 -DVIRT_MEMBERS $(abs_top_srcdir)/database/engine/journalfile_v2.ksy.in > $@
 endif
 
 API_PLUGIN_FILES = \
diff --git a/database/engine/journalfile.ksy b/database/engine/journalfile_v2.ksy.in
index 858db83d4b..6a656bc454 100644
--- a/database/engine/journalfile.ksy
+++ b/database/engine/journalfile_v2.ksy.in
@@ -1,6 +1,9 @@
 meta:
-  id: netdata_journalfile_v2
+  id: journalfile_v2`'ifdef(`VIRT_MEMBERS',`_virtmemb')
   endian: le
+  application: netdata
+  file-extension: njfv2
+  license: GPL-3.0-or-later
 
 seq:
   - id: journal_v2_header
@@ -19,12 +22,14 @@ seq:
   - id: metric_trailer
     type: journal_v2_block_trailer
   - id: page_blocs
-    type: jounral_v2_page_blocs
+    type: journal_v2_page_block
+    repeat: expr
+    repeat-expr: _root.journal_v2_header.metric_count
+  - id: padding
     size: _root._io.size - _root._io.pos - 4
   - id: journal_file_trailer
     type: journal_v2_block_trailer
 
-
 types:
   journal_v2_metric_list:
     seq:
@@ -38,11 +43,13 @@ types:
         type: u4
       - id: delta_end_s
         type: u4
-    instances:
+ifdef(`VIRT_MEMBERS',
+`    instances:
       page_block:
         type: journal_v2_page_block
         io: _root._io
         pos: page_offset
+')dnl
   journal_v2_page_hdr:
     seq:
       - id: crc
@@ -69,11 +76,13 @@ types:
         type: u1
       - id: reserved
         type: u1
-    instances:
+ifdef(`VIRT_MEMBERS',
+`    instances:
       extent:
         io: _root._io
         type: journal_v2_extent_list
         pos: _root.journal_v2_header.extent_offset + (extent_idx * 16)
+')dnl
   journal_v2_header:
     seq:
       - id: magic
@@ -106,11 +115,13 @@ types:
         type: u4
       - id: data
         type: u8
-    instances:
+ifdef(`VIRT_MEMBERS',
+`    instances:
       trailer:
         io: _root._io
         type: journal_v2_block_trailer
         pos: _root._io.size - 4
+')dnl
   journal_v2_block_trailer:
     seq:
       - id: checksum
@@ -137,8 +148,3 @@ types:
         repeat-expr: hdr.entries
       - id: block_trailer
         type: journal_v2_block_trailer
-  jounral_v2_page_blocs:
-    seq:
-      - id: blocs
-        type: journal_v2_page_block
-        repeat: eos