Export CSV config revamp to support table exports

meta/src/meta/grammar.y

@@ -99,9 +99,10 @@
 %nonterm exists logic.Exists
 %nonterm export transactions.Export
 %nonterm export_csv_column transactions.ExportCSVColumn
-%nonterm export_csv_columns Sequence[transactions.ExportCSVColumn]
+%nonterm export_csv_columns_list Sequence[transactions.ExportCSVColumn]
 %nonterm export_csv_config transactions.ExportCSVConfig
 %nonterm export_csv_path String
+%nonterm export_csv_source transactions.ExportCSVSource
 %nonterm false logic.Disjunction
 %nonterm ffi logic.FFI
 %nonterm ffi_args Sequence[logic.Abstraction]
@@ -178,6 +179,7 @@
 %validator_ignore_completeness DecimalValue
 %validator_ignore_completeness BeTreeLocator
 %validator_ignore_completeness BeTreeConfig
+%validator_ignore_completeness ExportCSVColumns
 
 %%
 
@@ -1079,17 +1081,23 @@ export
       deconstruct: $3: transactions.ExportCSVConfig = $$.csv_config
 
 export_csv_config
-    : "(" "export_csv_config" export_csv_path export_csv_columns config_dict ")"
-      construct: $$ = export_csv_config($3, $4, $5)
-      deconstruct:
+    : "(" "export_csv_config_v2" export_csv_path export_csv_source csv_config ")"
+      construct: $$ = construct_export_csv_config_with_source($3, $4, $5)
+      deconstruct if builtin.length($$.data_columns) == 0:
+        $3: String = $$.path
+        $4: transactions.ExportCSVSource = $$.csv_source
+        $5: logic.CSVConfig = $$.csv_config
+    | "(" "export_csv_config" export_csv_path export_csv_columns_list config_dict ")"
+      construct: $$ = construct_export_csv_config($3, $4, $5)
+      deconstruct if builtin.length($$.data_columns) != 0:
         $3: String = $$.path
         $4: Sequence[transactions.ExportCSVColumn] = $$.data_columns
         $5: Sequence[Tuple[String, logic.Value]] = deconstruct_export_csv_config($$)
 
 export_csv_path
     : "(" "path" STRING ")"
 
-export_csv_columns
+export_csv_columns_list
     : "(" "columns" export_csv_column* ")"
 
 export_csv_column
@@ -1099,6 +1107,16 @@ export_csv_column
         $3: String = $$.column_name
         $4: logic.RelationId = $$.column_data
 
+export_csv_source
+    : "(" "gnf_columns" export_csv_column* ")"
+      construct: $$ = transactions.ExportCSVSource(gnf_columns=transactions.ExportCSVColumns(columns=$3))
+      deconstruct if builtin.has_proto_field($$, 'gnf_columns'):
+        $3: Sequence[transactions.ExportCSVColumn] = $$.gnf_columns.columns
+    | "(" "table_def" relation_id ")"
+      construct: $$ = transactions.ExportCSVSource(table_def=$3)
+      deconstruct if builtin.has_proto_field($$, 'table_def'):
+        $3: logic.RelationId = $$.table_def
+
 
 %%
 
@@ -1198,6 +1216,7 @@ def construct_csv_config(config_dict: Sequence[Tuple[String, logic.Value]]) -> l
     decimal_separator: str = _extract_value_string(builtin.dict_get(config, "csv_decimal_separator"), ".")
     encoding: str = _extract_value_string(builtin.dict_get(config, "csv_encoding"), "utf-8")
     compression: str = _extract_value_string(builtin.dict_get(config, "csv_compression"), "auto")
+    partition_size_mb: int = _extract_value_int64(builtin.dict_get(config, "csv_partition_size_mb"), 0)
     return logic.CSVConfig(
         header_row=header_row,
         skip=skip,
@@ -1210,6 +1229,7 @@ def construct_csv_config(config_dict: Sequence[Tuple[String, logic.Value]]) -> l
         decimal_separator=decimal_separator,
         encoding=encoding,
         compression=compression,
+        partition_size_mb=partition_size_mb,
     )
 
 
@@ -1275,8 +1295,7 @@ def construct_configure(config_dict: Sequence[Tuple[String, logic.Value]]) -> tr
         ivm_config=ivm_config,
     )
 
-
-def export_csv_config(
+def construct_export_csv_config(
     path: String,
     columns: Sequence[transactions.ExportCSVColumn],
     config_dict: Sequence[Tuple[String, logic.Value]],
@@ -1301,6 +1320,17 @@ def export_csv_config(
         syntax_escapechar=builtin.some(syntax_escapechar),
     )
 
+def construct_export_csv_config_with_source(
+    path: String,
+    csv_source: transactions.ExportCSVSource,
+    csv_config: logic.CSVConfig,
+) -> transactions.ExportCSVConfig:
+    return transactions.ExportCSVConfig(
+        path=path,
+        csv_source=csv_source,
+        csv_config=csv_config,
+    )
+
 
 def _make_value_int32(v: Int32) -> logic.Value:
     return logic.Value(int_value=builtin.int32_to_int64(v))
@@ -1362,6 +1392,8 @@ def deconstruct_csv_config(msg: logic.CSVConfig) -> List[Tuple[String, logic.Val
     builtin.list_push(result, builtin.tuple("csv_decimal_separator", _make_value_string(msg.decimal_separator)))
     builtin.list_push(result, builtin.tuple("csv_encoding", _make_value_string(msg.encoding)))
     builtin.list_push(result, builtin.tuple("csv_compression", _make_value_string(msg.compression)))
+    if msg.partition_size_mb != 0:
+      builtin.list_push(result, builtin.tuple("csv_partition_size_mb", _make_value_int64(msg.partition_size_mb)))
     return builtin.list_sort(result)
 
 

proto/relationalai/lqp/v1/logic.proto

@@ -309,6 +309,9 @@ message CSVConfig {
 
   // Compression
   string compression = 11; // "none", "gzip", "zstd", "auto" (default: "auto")
+
+  // Partitioning (for export)
+  int64 partition_size_mb = 12;
 }
 
 message GNFColumn {

proto/relationalai/lqp/v1/transactions.proto

@@ -82,6 +82,11 @@ message Snapshot {
 
 message ExportCSVConfig {
   string path = 1;
+
+  ExportCSVSource csv_source = 10;
+  CSVConfig csv_config = 11;
+
+  // Below are all deprecated in favour of the `csv_config` above.
   repeated ExportCSVColumn data_columns = 2;
 
   optional int64 partition_size = 3;
@@ -100,6 +105,17 @@ message ExportCSVColumn {
   RelationId column_data = 2;
 }
 
+message ExportCSVColumns {
+  repeated ExportCSVColumn columns = 1;
+}
+
+message ExportCSVSource {
+  oneof csv_source {
+    ExportCSVColumns gnf_columns = 1;
+    RelationId table_def = 2;
+  }
+}
+
 //
 // Read operations
 //