From dd1b306ae632a06ae24ff1e987161b55aef4f257 Mon Sep 17 00:00:00 2001 From: Harel Ben-Attia Date: Sat, 6 Oct 2018 13:02:16 +0300 Subject: 20% improvement --- bin/q | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/bin/q b/bin/q index f6a1a02..56963c4 100755 --- a/bin/q +++ b/bin/q @@ -790,6 +790,7 @@ class TableCreator(object): self.pre_creation_rows = [] self.buffered_inserts = [] + self.effective_column_names = None # Column type indices for columns that contain numeric types. Lazily initialized # so column inferer can do its work before this information is needed @@ -934,6 +935,7 @@ class TableCreator(object): self.pre_creation_rows.append(col_vals) return + # The table already exists, so we can just add a new row self._insert_row_i(col_vals) @@ -993,13 +995,14 @@ class TableCreator(object): def _insert_row_i(self, col_vals): col_vals = self.normalize_col_vals(col_vals) - effective_column_names = self.column_inferer.column_names[ - :len(col_vals)] - if len(effective_column_names) > 0: - self.buffered_inserts.append((effective_column_names, col_vals)) + if self.effective_column_names is None: + self.effective_column_names = self.column_inferer.column_names[:len(col_vals)] + + if len(self.effective_column_names) > 0: + self.buffered_inserts.append(col_vals) else: - self.buffered_inserts.append((["c1"], [""])) + self.buffered_inserts.append([""]) if len(self.buffered_inserts) < 5000: return @@ -1014,10 +1017,9 @@ class TableCreator(object): if len(self.buffered_inserts) > 0: insert_row_stmt = self.db.generate_insert_row( - self.table_name, self.buffered_inserts[0][0]) - params = [col_vals for col_names, col_vals in self.buffered_inserts] + self.table_name, self.effective_column_names) - self.db.update_many(insert_row_stmt, params) + self.db.update_many(insert_row_stmt, self.buffered_inserts) # print self.db.execute_and_fetch(self.db.generate_end_transaction()) self.buffered_inserts = [] -- cgit v1.2.3