From 8f8d967c2277546cdec740702dc21fa3063b0025 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 23 Jul 2024 15:48:47 +0200 Subject: [PATCH 1/7] Rollup node: tweak encoding of snapshot metadata --- .../main_smart_rollup_node.ml | 3 +- src/lib_smart_rollup_node/snapshot_utils.ml | 48 ++++++++----------- src/lib_smart_rollup_node/snapshot_utils.mli | 1 + src/lib_smart_rollup_node/snapshots.ml | 1 + 4 files changed, 24 insertions(+), 29 deletions(-) diff --git a/src/bin_smart_rollup_node/main_smart_rollup_node.ml b/src/bin_smart_rollup_node/main_smart_rollup_node.ml index 1cc44e9d7995..451bc2c90f99 100644 --- a/src/bin_smart_rollup_node/main_smart_rollup_node.ml +++ b/src/bin_smart_rollup_node/main_smart_rollup_node.ml @@ -515,7 +515,8 @@ let snapshot_info = (prefixes ["snapshot"; "info"] @@ Cli.snapshot_file_param @@ stop) (fun () snapshot_file cctxt -> let open Lwt_result_syntax in - let ( {Snapshot_utils.history_mode; address; head_level; last_commitment}, + let ( Snapshot_utils. + {version = _; history_mode; address; head_level; last_commitment}, compressed ) = Snapshots.info ~snapshot_file in diff --git a/src/lib_smart_rollup_node/snapshot_utils.ml b/src/lib_smart_rollup_node/snapshot_utils.ml index 42d273d8c0f7..943724d474fb 100644 --- a/src/lib_smart_rollup_node/snapshot_utils.ml +++ b/src/lib_smart_rollup_node/snapshot_utils.ml @@ -73,32 +73,30 @@ let gzip_writer : writer = (module Gzip_writer) type snapshot_version = V0 type snapshot_metadata = { + version : snapshot_version; history_mode : Configuration.history_mode; address : Address.t; head_level : int32; last_commitment : Commitment.Hash.t; } -let snapshot_version_encoding = - let open Data_encoding in - conv_with_guard - (function V0 -> 0) - (function - | 0 -> Ok V0 | x -> Error ("Invalid snapshot version " ^ string_of_int x)) - int8 - let snaphsot_metadata_encoding = let open Data_encoding in - conv - (fun {history_mode; address; head_level; last_commitment} -> - (history_mode, address, head_level, last_commitment)) - (fun (history_mode, address, head_level, last_commitment) -> - {history_mode; address; head_level; last_commitment}) - @@ obj4 - (req "history_mode" Configuration.history_mode_encoding) - (req "address" Address.encoding) - (req "head_level" int32) - (req "last_commitment" Commitment.Hash.encoding) + union + [ + case + ~title:"snapshot_metadata_v0" + (Tag 0) + (obj4 + (req "history_mode" Configuration.history_mode_encoding) + (req "address" Address.encoding) + (req "head_level" int32) + (req "last_commitment" Commitment.Hash.encoding)) + (fun {version = V0; history_mode; address; head_level; last_commitment} -> + Some (history_mode, address, head_level, last_commitment)) + (fun (history_mode, address, head_level, last_commitment) -> + {version = V0; history_mode; address; head_level; last_commitment}); + ] let snapshot_metadata_size = Data_encoding.Binary.fixed_length snaphsot_metadata_encoding @@ -107,25 +105,19 @@ let snapshot_metadata_size = let version = V0 let write_snapshot_metadata (module Writer : WRITER_OUTPUT) metadata = - let version_bytes = - Data_encoding.Binary.to_bytes_exn snapshot_version_encoding version - in let metadata_bytes = Data_encoding.Binary.to_bytes_exn snaphsot_metadata_encoding metadata in - Writer.output Writer.out_chan version_bytes 0 (Bytes.length version_bytes) ; Writer.output Writer.out_chan metadata_bytes 0 (Bytes.length metadata_bytes) let read_snapshot_metadata (module Reader : READER_INPUT) = - let version_bytes = Bytes.create 1 in let metadata_bytes = Bytes.create snapshot_metadata_size in - Reader.really_input Reader.in_chan version_bytes 0 1 ; Reader.really_input Reader.in_chan metadata_bytes 0 snapshot_metadata_size ; - let snapshot_version = - Data_encoding.Binary.of_bytes_exn snapshot_version_encoding version_bytes + let metadata = + Data_encoding.Binary.of_bytes_exn snaphsot_metadata_encoding metadata_bytes in - assert (snapshot_version = version) ; - Data_encoding.Binary.of_bytes_exn snaphsot_metadata_encoding metadata_bytes + assert (metadata.version = version) ; + metadata let create (module Reader : READER) (module Writer : WRITER) metadata ~dir ~include_file ~dest = diff --git a/src/lib_smart_rollup_node/snapshot_utils.mli b/src/lib_smart_rollup_node/snapshot_utils.mli index 79e47d3a4ec0..83795e6fda2b 100644 --- a/src/lib_smart_rollup_node/snapshot_utils.mli +++ b/src/lib_smart_rollup_node/snapshot_utils.mli @@ -31,6 +31,7 @@ type snapshot_version = V0 (** Snapshot metadata for version 0. This information is written as a header of the archive snapshot file. *) type snapshot_metadata = { + version : snapshot_version; history_mode : Configuration.history_mode; address : Address.t; head_level : int32; diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index d64363d4c0eb..d513370b2865 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -121,6 +121,7 @@ let pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir = let* () = Store.close store in return { + version = V0; history_mode; address = metadata.rollup_address; head_level = head.header.level; -- GitLab From 26a2d2464242e74ad3e547d58eea5fdc5443573d Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 23 Jul 2024 16:07:53 +0200 Subject: [PATCH 2/7] Rollup node: parameterized snapshot utils by metadata type --- .../main_smart_rollup_node.ml | 2 +- src/lib_smart_rollup_node/snapshot_utils.ml | 373 ++++++++---------- src/lib_smart_rollup_node/snapshot_utils.mli | 81 ++-- src/lib_smart_rollup_node/snapshots.ml | 54 ++- src/lib_smart_rollup_node/snapshots.mli | 21 +- 5 files changed, 279 insertions(+), 252 deletions(-) diff --git a/src/bin_smart_rollup_node/main_smart_rollup_node.ml b/src/bin_smart_rollup_node/main_smart_rollup_node.ml index 451bc2c90f99..558119b8a206 100644 --- a/src/bin_smart_rollup_node/main_smart_rollup_node.ml +++ b/src/bin_smart_rollup_node/main_smart_rollup_node.ml @@ -515,7 +515,7 @@ let snapshot_info = (prefixes ["snapshot"; "info"] @@ Cli.snapshot_file_param @@ stop) (fun () snapshot_file cctxt -> let open Lwt_result_syntax in - let ( Snapshot_utils. + let ( Snapshots.Snapshot_metadata. {version = _; history_mode; address; head_level; last_commitment}, compressed ) = Snapshots.info ~snapshot_file diff --git a/src/lib_smart_rollup_node/snapshot_utils.ml b/src/lib_smart_rollup_node/snapshot_utils.ml index 943724d474fb..1e3252b06296 100644 --- a/src/lib_smart_rollup_node/snapshot_utils.ml +++ b/src/lib_smart_rollup_node/snapshot_utils.ml @@ -70,220 +70,191 @@ let gzip_reader : reader = (module Gzip_reader) let gzip_writer : writer = (module Gzip_writer) -type snapshot_version = V0 - -type snapshot_metadata = { - version : snapshot_version; - history_mode : Configuration.history_mode; - address : Address.t; - head_level : int32; - last_commitment : Commitment.Hash.t; -} - -let snaphsot_metadata_encoding = - let open Data_encoding in - union - [ - case - ~title:"snapshot_metadata_v0" - (Tag 0) - (obj4 - (req "history_mode" Configuration.history_mode_encoding) - (req "address" Address.encoding) - (req "head_level" int32) - (req "last_commitment" Commitment.Hash.encoding)) - (fun {version = V0; history_mode; address; head_level; last_commitment} -> - Some (history_mode, address, head_level, last_commitment)) - (fun (history_mode, address, head_level, last_commitment) -> - {version = V0; history_mode; address; head_level; last_commitment}); - ] - -let snapshot_metadata_size = - Data_encoding.Binary.fixed_length snaphsot_metadata_encoding - |> WithExceptions.Option.get ~loc:__LOC__ - -let version = V0 - -let write_snapshot_metadata (module Writer : WRITER_OUTPUT) metadata = - let metadata_bytes = - Data_encoding.Binary.to_bytes_exn snaphsot_metadata_encoding metadata - in - Writer.output Writer.out_chan metadata_bytes 0 (Bytes.length metadata_bytes) - -let read_snapshot_metadata (module Reader : READER_INPUT) = - let metadata_bytes = Bytes.create snapshot_metadata_size in - Reader.really_input Reader.in_chan metadata_bytes 0 snapshot_metadata_size ; - let metadata = - Data_encoding.Binary.of_bytes_exn snaphsot_metadata_encoding metadata_bytes - in - assert (metadata.version = version) ; - metadata - -let create (module Reader : READER) (module Writer : WRITER) metadata ~dir - ~include_file ~dest = - let module Archive_writer = Tar.Make (struct - include Reader - include Writer - end) in - let files = - Tezos_stdlib_unix.Utils.list_files dir - |> List.filter (fun relative_path -> include_file ~relative_path) - in - let total = - List.fold_left - (fun total relative_path -> - let {Unix.st_size; _} = - Unix.lstat (Filename.concat dir relative_path) +module Make (Snapshot_metadata : sig + type t + + val encoding : t Data_encoding.t + + val size : int +end) = +struct + let write_snapshot_metadata (module Writer : WRITER_OUTPUT) metadata = + let metadata_bytes = + Data_encoding.Binary.to_bytes_exn Snapshot_metadata.encoding metadata + in + Writer.output Writer.out_chan metadata_bytes 0 (Bytes.length metadata_bytes) + + let read_snapshot_metadata (module Reader : READER_INPUT) = + let metadata_bytes = Bytes.create Snapshot_metadata.size in + Reader.really_input Reader.in_chan metadata_bytes 0 Snapshot_metadata.size ; + Data_encoding.Binary.of_bytes_exn Snapshot_metadata.encoding metadata_bytes + + let create (module Reader : READER) (module Writer : WRITER) metadata ~dir + ~include_file ~dest = + let module Archive_writer = Tar.Make (struct + include Reader + include Writer + end) in + let files = + Tezos_stdlib_unix.Utils.list_files dir + |> List.filter (fun relative_path -> include_file ~relative_path) + in + let total = + List.fold_left + (fun total relative_path -> + let {Unix.st_size; _} = + Unix.lstat (Filename.concat dir relative_path) + in + total + st_size) + 0 + files + in + let progress_bar = + Progress_bar.progress_bar + ~counter:`Bytes + ~message:"Exporting snapshot " + ~color:(Terminal.Color.rgb 3 132 252) + total + in + Progress_bar.with_reporter progress_bar @@ fun count_progress -> + let write_file file (out_chan : Writer.out_channel) = + let in_chan = Reader.open_in file in + try + let buffer_size = 64 * 1024 in + let buf = Bytes.create buffer_size in + let rec copy () = + let read_bytes = Reader.input in_chan buf 0 buffer_size in + Writer.output out_chan buf 0 read_bytes ; + count_progress read_bytes ; + if read_bytes > 0 then copy () in - total + st_size) - 0 - files - in - let progress_bar = - Progress_bar.progress_bar - ~counter:`Bytes - ~message:"Exporting snapshot " - ~color:(Terminal.Color.rgb 3 132 252) - total - in - Progress_bar.with_reporter progress_bar @@ fun count_progress -> - let write_file file (out_chan : Writer.out_channel) = - let in_chan = Reader.open_in file in + copy () ; + Writer.flush_continue out_chan ; + Reader.close_in in_chan + with e -> + Reader.close_in in_chan ; + raise e + in + let file_stream = + List.rev_map + (fun relative_path -> + let full_path = Filename.concat dir relative_path in + let {Unix.st_perm; st_size; st_mtime; _} = Unix.lstat full_path in + let header = + Tar.Header.make + ~file_mode:st_perm + ~mod_time:(Int64.of_float st_mtime) + relative_path + (Int64.of_int st_size) + in + let writer = write_file full_path in + (header, writer)) + files + |> Stream.of_list + in + let out_chan = Writer.open_out dest in + try + write_snapshot_metadata + (module struct + include Writer + + let out_chan = out_chan + end) + metadata ; + Archive_writer.Archive.create_gen file_stream out_chan ; + Writer.close_out out_chan + with e -> + Writer.close_out out_chan ; + raise e + + let extract (module Reader : READER) (module Writer : WRITER) metadata_check + ~snapshot_file ~dest = + let open Lwt_result_syntax in + let module Writer = struct + include Writer + + let count_progress = ref (fun _ -> ()) + + let output oc b p l = + !count_progress 1 ; + output oc b p l + end in + let module Archive_reader = Tar.Make (struct + include Reader + include Writer + end) in + let out_channel_of_header (header : Tar.Header.t) = + let path = Filename.concat dest header.file_name in + Tezos_stdlib_unix.Utils.create_dir (Filename.dirname path) ; + Writer.open_out path + in + let in_chan = Reader.open_in snapshot_file in + let reader_input : (module READER_INPUT) = + (module struct + include Reader + + let in_chan = in_chan + end) + in + Lwt.finalize + (fun () -> + let metadata = read_snapshot_metadata reader_input in + let* check_result = metadata_check metadata in + let spinner = Progress_bar.spinner ~message:"Extracting snapshot" in + Progress_bar.with_reporter spinner @@ fun count_progress -> + Writer.count_progress := count_progress ; + Archive_reader.Archive.extract_gen out_channel_of_header in_chan ; + return (metadata, check_result)) + (fun () -> + Reader.close_in in_chan ; + Lwt.return_unit) + + let compress ~snapshot_file = + let Unix.{st_size = total; _} = Unix.stat snapshot_file in + let progress_bar = + Progress_bar.progress_bar + ~counter:`Bytes + ~message:"Compressing snapshot" + ~color:(Terminal.Color.rgb 3 198 252) + total + in + Progress_bar.with_reporter progress_bar @@ fun count_progress -> + let snapshot_file_gz = Filename.chop_suffix snapshot_file ".uncompressed" in + let in_chan = open_in snapshot_file in + let out_chan = Gzip.open_out snapshot_file_gz in try let buffer_size = 64 * 1024 in let buf = Bytes.create buffer_size in let rec copy () = - let read_bytes = Reader.input in_chan buf 0 buffer_size in - Writer.output out_chan buf 0 read_bytes ; + let read_bytes = input in_chan buf 0 buffer_size in + Gzip.output out_chan buf 0 read_bytes ; count_progress read_bytes ; if read_bytes > 0 then copy () in copy () ; - Writer.flush_continue out_chan ; - Reader.close_in in_chan + Gzip.close_out out_chan ; + close_in in_chan ; + Unix.unlink snapshot_file ; + snapshot_file_gz with e -> - Reader.close_in in_chan ; + Gzip.close_out out_chan ; + close_in in_chan ; raise e - in - let file_stream = - List.rev_map - (fun relative_path -> - let full_path = Filename.concat dir relative_path in - let {Unix.st_perm; st_size; st_mtime; _} = Unix.lstat full_path in - let header = - Tar.Header.make - ~file_mode:st_perm - ~mod_time:(Int64.of_float st_mtime) - relative_path - (Int64.of_int st_size) - in - let writer = write_file full_path in - (header, writer)) - files - |> Stream.of_list - in - let out_chan = Writer.open_out dest in - try - write_snapshot_metadata + + let read_metadata (module Reader : READER) ~snapshot_file = + let in_chan = Reader.open_in snapshot_file in + let reader_input : (module READER_INPUT) = (module struct - include Writer + include Reader - let out_chan = out_chan + let in_chan = in_chan end) - metadata ; - Archive_writer.Archive.create_gen file_stream out_chan ; - Writer.close_out out_chan - with e -> - Writer.close_out out_chan ; - raise e - -let extract (module Reader : READER) (module Writer : WRITER) metadata_check - ~snapshot_file ~dest = - let open Lwt_result_syntax in - let module Writer = struct - include Writer - - let count_progress = ref (fun _ -> ()) - - let output oc b p l = - !count_progress 1 ; - output oc b p l - end in - let module Archive_reader = Tar.Make (struct - include Reader - include Writer - end) in - let out_channel_of_header (header : Tar.Header.t) = - let path = Filename.concat dest header.file_name in - Tezos_stdlib_unix.Utils.create_dir (Filename.dirname path) ; - Writer.open_out path - in - let in_chan = Reader.open_in snapshot_file in - let reader_input : (module READER_INPUT) = - (module struct - include Reader - - let in_chan = in_chan - end) - in - Lwt.finalize - (fun () -> + in + try let metadata = read_snapshot_metadata reader_input in - let* check_result = metadata_check metadata in - let spinner = Progress_bar.spinner ~message:"Extracting snapshot" in - Progress_bar.with_reporter spinner @@ fun count_progress -> - Writer.count_progress := count_progress ; - Archive_reader.Archive.extract_gen out_channel_of_header in_chan ; - return (metadata, check_result)) - (fun () -> Reader.close_in in_chan ; - Lwt.return_unit) - -let compress ~snapshot_file = - let Unix.{st_size = total; _} = Unix.stat snapshot_file in - let progress_bar = - Progress_bar.progress_bar - ~counter:`Bytes - ~message:"Compressing snapshot" - ~color:(Terminal.Color.rgb 3 198 252) - total - in - Progress_bar.with_reporter progress_bar @@ fun count_progress -> - let snapshot_file_gz = Filename.chop_suffix snapshot_file ".uncompressed" in - let in_chan = open_in snapshot_file in - let out_chan = Gzip.open_out snapshot_file_gz in - try - let buffer_size = 64 * 1024 in - let buf = Bytes.create buffer_size in - let rec copy () = - let read_bytes = input in_chan buf 0 buffer_size in - Gzip.output out_chan buf 0 read_bytes ; - count_progress read_bytes ; - if read_bytes > 0 then copy () - in - copy () ; - Gzip.close_out out_chan ; - close_in in_chan ; - Unix.unlink snapshot_file ; - snapshot_file_gz - with e -> - Gzip.close_out out_chan ; - close_in in_chan ; - raise e - -let read_metadata (module Reader : READER) ~snapshot_file = - let in_chan = Reader.open_in snapshot_file in - let reader_input : (module READER_INPUT) = - (module struct - include Reader - - let in_chan = in_chan - end) - in - try - let metadata = read_snapshot_metadata reader_input in - Reader.close_in in_chan ; - metadata - with e -> - Reader.close_in in_chan ; - raise e + metadata + with e -> + Reader.close_in in_chan ; + raise e +end diff --git a/src/lib_smart_rollup_node/snapshot_utils.mli b/src/lib_smart_rollup_node/snapshot_utils.mli index 83795e6fda2b..d7cd3fd4f104 100644 --- a/src/lib_smart_rollup_node/snapshot_utils.mli +++ b/src/lib_smart_rollup_node/snapshot_utils.mli @@ -25,51 +25,46 @@ val gzip_reader : reader (** A writer for compressed files or snapshot archives. *) val gzip_writer : writer -(** Versioning of snapshot format. Only one version for now. *) -type snapshot_version = V0 +module Make (Snapshot_metadata : sig + type t -(** Snapshot metadata for version 0. This information is written as a header of - the archive snapshot file. *) -type snapshot_metadata = { - version : snapshot_version; - history_mode : Configuration.history_mode; - address : Address.t; - head_level : int32; - last_commitment : Commitment.Hash.t; -} + val encoding : t Data_encoding.t -(** [create reader writer metadata ~dir ~include_file ~dest] creates a snapshot - archive with the header [metadata] and the hierarchy of files in directory - [dir] for which [include_file] returns true. The archive is produced in file - [dest]. *) -val create : - reader -> - writer -> - snapshot_metadata -> - dir:string -> - include_file:(relative_path:string -> bool) -> - dest:string -> - unit + val size : int +end) : sig + (** [create reader writer metadata ~dir ~include_file ~dest] creates a + snapshot archive with the header [metadata] and the hierarchy of files in + directory [dir] for which [include_file] returns true. The archive is + produced in file [dest]. *) + val create : + reader -> + writer -> + Snapshot_metadata.t -> + dir:string -> + include_file:(relative_path:string -> bool) -> + dest:string -> + unit -(** [extract reader writer check_metadata ~snapshot_file ~dest] extracts the - snapshot archive [snapshot_file] in the directory [dest]. Existing files in - [dest] with the same names are overwritten. The metadata header read from - the snapshot is checked with [check_metadata] before beginning - extraction, and returned. *) -val extract : - reader -> - writer -> - (snapshot_metadata -> 'a tzresult Lwt.t) -> - snapshot_file:string -> - dest:string -> - (snapshot_metadata * 'a) tzresult Lwt.t + (** [extract reader writer check_metadata ~snapshot_file ~dest] extracts the + snapshot archive [snapshot_file] in the directory [dest]. Existing files + in [dest] with the same names are overwritten. The metadata header read + from the snapshot is checked with [check_metadata] before beginning + extraction, and returned. *) + val extract : + reader -> + writer -> + (Snapshot_metadata.t -> 'a tzresult Lwt.t) -> + snapshot_file:string -> + dest:string -> + (Snapshot_metadata.t * 'a) tzresult Lwt.t -(** [compress ~snapshot_file] compresses the snapshot archive [snapshot_file] of - the form ["path/to/snapshot.uncompressed"] to a new file - ["path/to/snapshot"] whose path is returned. [snapshot_file] is removed upon - successful compression. *) -val compress : snapshot_file:string -> string + (** [compress ~snapshot_file] compresses the snapshot archive [snapshot_file] + of the form ["path/to/snapshot.uncompressed"] to a new file + ["path/to/snapshot"] whose path is returned. [snapshot_file] is removed + upon successful compression. *) + val compress : snapshot_file:string -> string -(** [read_metadata reader ~snapshot_file] reads the metadata from the snapshot - file without extracting it. *) -val read_metadata : reader -> snapshot_file:string -> snapshot_metadata + (** [read_metadata reader ~snapshot_file] reads the metadata from the snapshot + file without extracting it. *) + val read_metadata : reader -> snapshot_file:string -> Snapshot_metadata.t +end diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index d513370b2865..466121a27e47 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -9,6 +9,47 @@ open Snapshot_utils type compression = No | On_the_fly | After +module Snapshot_metadata = struct + type version = V0 + + type t = { + version : version; + history_mode : Configuration.history_mode; + address : Address.t; + head_level : int32; + last_commitment : Commitment.Hash.t; + } + + let encoding = + let open Data_encoding in + union + [ + case + ~title:"snapshot_metadata_v0" + (Tag 0) + (obj4 + (req "history_mode" Configuration.history_mode_encoding) + (req "address" Address.encoding) + (req "head_level" int32) + (req "last_commitment" Commitment.Hash.encoding)) + (fun { + version = V0; + history_mode; + address; + head_level; + last_commitment; + } -> Some (history_mode, address, head_level, last_commitment)) + (fun (history_mode, address, head_level, last_commitment) -> + {version = V0; history_mode; address; head_level; last_commitment}); + ] + + let size = + Data_encoding.Binary.fixed_length encoding + |> WithExceptions.Option.get ~loc:__LOC__ +end + +open Snapshot_utils.Make (Snapshot_metadata) + let check_store_version store_dir = let open Lwt_result_syntax in let* store_version = Store_version.read_version_file ~dir:store_dir in @@ -121,7 +162,7 @@ let pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir = let* () = Store.close store in return { - version = V0; + Snapshot_metadata.version = V0; history_mode; address = metadata.rollup_address; head_level = head.header.level; @@ -378,7 +419,7 @@ let check_l2_chain ~message ~data_dir (store : _ Store.t) context in check_chain head.header.block_hash None -let check_last_commitment head snapshot_metadata = +let check_last_commitment head (snapshot_metadata : Snapshot_metadata.t) = let last_snapshot_commitment = Sc_rollup_block.most_recent_commitment head.Sc_rollup_block.header in @@ -672,7 +713,8 @@ let with_locks ~data_dir f = ~filename:(Node_context.processing_lockfile_path ~data_dir) f -let export_dir metadata ~take_locks ~compression ~data_dir ~dest ~filename = +let export_dir (metadata : Snapshot_metadata.t) ~take_locks ~compression + ~data_dir ~dest ~filename = let open Lwt_result_syntax in let* snapshot_file = let with_locks = @@ -835,7 +877,8 @@ let export_compact cctxt ~no_checks ~compression ~data_dir ~dest ~filename = ~dest ~filename -let pre_import_checks cctxt ~no_checks ~data_dir snapshot_metadata = +let pre_import_checks cctxt ~no_checks ~data_dir + (snapshot_metadata : Snapshot_metadata.t) = let open Lwt_result_syntax in let store_dir = Configuration.default_storage_dir data_dir in (* Load stores in read-only to make simple checks. *) @@ -917,7 +960,8 @@ let check_data_dir_unpopulated data_dir () = data_dir else return_unit -let maybe_gc_after_import cctxt ~data_dir snapshot_metadata +let maybe_gc_after_import cctxt ~data_dir + (snapshot_metadata : Snapshot_metadata.t) (original_history_mode : Configuration.history_mode option) = let open Lwt_result_syntax in match (original_history_mode, snapshot_metadata.history_mode) with diff --git a/src/lib_smart_rollup_node/snapshots.mli b/src/lib_smart_rollup_node/snapshots.mli index 35c47e337949..22015a8ba7e6 100644 --- a/src/lib_smart_rollup_node/snapshots.mli +++ b/src/lib_smart_rollup_node/snapshots.mli @@ -17,6 +17,24 @@ type compression = temporarily than {!On_the_fly} but does not lock the rollup node for very long. *) +module Snapshot_metadata : sig + (** Versioning of snapshot format. Only one version for now. *) + type version = V0 + + (** Snapshot metadata for version 0. This information is written as a header of + the archive snapshot file. *) + type t = { + version : version; + history_mode : Configuration.history_mode; + address : Address.t; + head_level : int32; + last_commitment : Commitment.Hash.t; + } + + (** Fixed size metadata encoding. *) + val encoding : t Data_encoding.t +end + (** [export cctxt ~no_checks ~compression ~data_dir ~dest ~filename] creates a tar gzipped archive with name [filename] (or a generated name) in [dest] (or the current directory) containing a snapshot of the data of the rollup node @@ -68,8 +86,7 @@ val import : (** [info ~snapshot_file] returns information that can be used to inspect the snapshot file. *) val info : - snapshot_file:string -> - Snapshot_utils.snapshot_metadata * [`Compressed | `Uncompressed] + snapshot_file:string -> Snapshot_metadata.t * [`Compressed | `Uncompressed] (** [with_modify_data_dir cctxt ~data_dir ~apply_unsafe_patches ?skip_condition f] applies [f] in a read-write context that is created from [data-dir] (and -- GitLab From fe0461c72e183fa738106acbdfc0b1ba8a2d00ba Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 23 Jul 2024 16:10:10 +0200 Subject: [PATCH 3/7] Build: camlzip and tar deps in lib_layer2_store --- manifest/product_octez.ml | 6 +++--- opam/octez-l2-libs.opam | 3 +++ opam/octez-smart-rollup-node-lib.opam | 3 --- src/lib_layer2_store/dune | 5 ++++- src/lib_smart_rollup_node/dune | 3 --- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index b1adcae5114a..53cabe9bb4cf 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -4407,6 +4407,9 @@ let octez_layer2_store = octez_context_sigs; octez_context_helpers; octez_riscv_pvm; + camlzip; + tar; + tar_unix; ] ~linkall:true ~conflicts:[Conflicts.checkseum] @@ -4669,9 +4672,6 @@ let octez_smart_rollup_node_lib = octez_openapi; octez_node_config; prometheus_app; - camlzip; - tar; - tar_unix; octez_dal_node_lib |> open_; octez_dac_lib |> open_; octez_dac_client_lib |> open_; diff --git a/opam/octez-l2-libs.opam b/opam/octez-l2-libs.opam index 31689d673a67..80f0303e3298 100644 --- a/opam/octez-l2-libs.opam +++ b/opam/octez-l2-libs.opam @@ -21,6 +21,9 @@ depends: [ "octez-internal-libs" { = version } "aches-lwt" { >= "1.0.0" } "octez-riscv-pvm" { = version } + "camlzip" { >= "1.11" & < "1.12" } + "tar" + "tar-unix" { >= "2.0.1" & < "3.0.0" } "yaml" { >= "3.1.0" } "ppx_import" "qcheck-alcotest" { >= "0.20" } diff --git a/opam/octez-smart-rollup-node-lib.opam b/opam/octez-smart-rollup-node-lib.opam index 987304982819..9ecbd9602793 100644 --- a/opam/octez-smart-rollup-node-lib.opam +++ b/opam/octez-smart-rollup-node-lib.opam @@ -15,9 +15,6 @@ depends: [ "cohttp-lwt-unix" { = "5.3.0" } "tezos-openapi" { = version } "octez-node-config" { = version } - "camlzip" { >= "1.11" & < "1.12" } - "tar" - "tar-unix" { >= "2.0.1" & < "3.0.0" } "tezos-dal-node-lib" { = version } "tezos-dac-lib" { = version } "tezos-dac-client-lib" { = version } diff --git a/src/lib_layer2_store/dune b/src/lib_layer2_store/dune index 7006308d99ac..e3aabc7b2d2e 100644 --- a/src/lib_layer2_store/dune +++ b/src/lib_layer2_store/dune @@ -16,7 +16,10 @@ octez-libs.tezos-context.encoding octez-libs.context.sigs octez-libs.tezos-context.helpers - octez-riscv-pvm) + octez-riscv-pvm + camlzip + tar + tar-unix) (library_flags (:standard -linkall)) (flags (:standard) diff --git a/src/lib_smart_rollup_node/dune b/src/lib_smart_rollup_node/dune index f7b757fa02cb..1300e4d26694 100644 --- a/src/lib_smart_rollup_node/dune +++ b/src/lib_smart_rollup_node/dune @@ -16,9 +16,6 @@ tezos-openapi octez-node-config octez-libs.prometheus-app - camlzip - tar - tar-unix tezos-dal-node-lib tezos-dac-lib tezos-dac-client-lib -- GitLab From 63b1dd3ea42a60c990ede39aae14580646be4876 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 23 Jul 2024 16:11:15 +0200 Subject: [PATCH 4/7] Layer2: move snapshot utils in lib_layer2_store --- .../snapshot_utils.ml | 2 +- .../snapshot_utils.mli | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/{lib_smart_rollup_node => lib_layer2_store}/snapshot_utils.ml (98%) rename src/{lib_smart_rollup_node => lib_layer2_store}/snapshot_utils.mli (96%) diff --git a/src/lib_smart_rollup_node/snapshot_utils.ml b/src/lib_layer2_store/snapshot_utils.ml similarity index 98% rename from src/lib_smart_rollup_node/snapshot_utils.ml rename to src/lib_layer2_store/snapshot_utils.ml index 1e3252b06296..acdf1f0b61d2 100644 --- a/src/lib_smart_rollup_node/snapshot_utils.ml +++ b/src/lib_layer2_store/snapshot_utils.ml @@ -1,7 +1,7 @@ (*****************************************************************************) (* *) (* SPDX-License-Identifier: MIT *) -(* Copyright (c) 2023 Functori *) +(* SPDX-FileCopyrightText: 2023-2024 Functori *) (* *) (*****************************************************************************) diff --git a/src/lib_smart_rollup_node/snapshot_utils.mli b/src/lib_layer2_store/snapshot_utils.mli similarity index 96% rename from src/lib_smart_rollup_node/snapshot_utils.mli rename to src/lib_layer2_store/snapshot_utils.mli index d7cd3fd4f104..3e4365ff6743 100644 --- a/src/lib_smart_rollup_node/snapshot_utils.mli +++ b/src/lib_layer2_store/snapshot_utils.mli @@ -1,7 +1,7 @@ (*****************************************************************************) (* *) (* SPDX-License-Identifier: MIT *) -(* Copyright (c) 2023 Functori *) +(* SPDX-FileCopyrightText: 2023-2024 Functori *) (* *) (*****************************************************************************) -- GitLab From 9e9312f10a3296e1168652a417cd12df7a02b7db Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Thu, 25 Jul 2024 14:58:24 +0200 Subject: [PATCH 5/7] L2 snapshots: put is_compressed_snapshot in utils library --- src/lib_layer2_store/snapshot_utils.ml | 15 +++++++++++++++ src/lib_layer2_store/snapshot_utils.mli | 4 ++++ src/lib_smart_rollup_node/snapshots.ml | 15 --------------- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/lib_layer2_store/snapshot_utils.ml b/src/lib_layer2_store/snapshot_utils.ml index acdf1f0b61d2..545a5ac00afe 100644 --- a/src/lib_layer2_store/snapshot_utils.ml +++ b/src/lib_layer2_store/snapshot_utils.ml @@ -70,6 +70,21 @@ let gzip_reader : reader = (module Gzip_reader) let gzip_writer : writer = (module Gzip_writer) +(* Magic bytes for gzip files is 1f8b. *) +let is_compressed_snapshot snapshot_file = + let ic = open_in snapshot_file in + try + let ok = input_byte ic = 0x1f && input_byte ic = 0x8b in + close_in ic ; + ok + with + | End_of_file -> + close_in ic ; + false + | e -> + close_in ic ; + raise e + module Make (Snapshot_metadata : sig type t diff --git a/src/lib_layer2_store/snapshot_utils.mli b/src/lib_layer2_store/snapshot_utils.mli index 3e4365ff6743..398f7870a130 100644 --- a/src/lib_layer2_store/snapshot_utils.mli +++ b/src/lib_layer2_store/snapshot_utils.mli @@ -25,6 +25,10 @@ val gzip_reader : reader (** A writer for compressed files or snapshot archives. *) val gzip_writer : writer +(** [is_compressed_snapshot f] returns [true] if [f] is the path of a compressed + snapshot file, i.e. a gzip file. *) +val is_compressed_snapshot : string -> bool + module Make (Snapshot_metadata : sig type t diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index 466121a27e47..59875029ef4d 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -664,21 +664,6 @@ let post_checks ?(apply_unsafe_patches = false) ~action ~message let* () = Store.close store in return_unit -(* Magic bytes for gzip files is 1f8b. *) -let is_compressed_snapshot snapshot_file = - let ic = open_in snapshot_file in - try - let ok = input_byte ic = 0x1f && input_byte ic = 0x8b in - close_in ic ; - ok - with - | End_of_file -> - close_in ic ; - false - | e -> - close_in ic ; - raise e - let post_export_checks ~snapshot_file = let open Lwt_result_syntax in Lwt_utils_unix.with_tempdir "snapshot_checks_" @@ fun dest -> -- GitLab From 158f984a8b4fdc1d10a0d890f34a079c9cd0acb8 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 24 Jul 2024 10:46:25 +0200 Subject: [PATCH 6/7] Snapshots: rename "snapshot metadata" to "snapshot header" --- .../main_smart_rollup_node.ml | 2 +- src/lib_layer2_store/snapshot_utils.ml | 38 ++++---- src/lib_layer2_store/snapshot_utils.mli | 22 ++--- src/lib_smart_rollup_node/snapshots.ml | 97 +++++++++---------- src/lib_smart_rollup_node/snapshots.mli | 5 +- 5 files changed, 81 insertions(+), 83 deletions(-) diff --git a/src/bin_smart_rollup_node/main_smart_rollup_node.ml b/src/bin_smart_rollup_node/main_smart_rollup_node.ml index 558119b8a206..8349fc0c6a07 100644 --- a/src/bin_smart_rollup_node/main_smart_rollup_node.ml +++ b/src/bin_smart_rollup_node/main_smart_rollup_node.ml @@ -515,7 +515,7 @@ let snapshot_info = (prefixes ["snapshot"; "info"] @@ Cli.snapshot_file_param @@ stop) (fun () snapshot_file cctxt -> let open Lwt_result_syntax in - let ( Snapshots.Snapshot_metadata. + let ( Snapshots.Header. {version = _; history_mode; address; head_level; last_commitment}, compressed ) = Snapshots.info ~snapshot_file diff --git a/src/lib_layer2_store/snapshot_utils.ml b/src/lib_layer2_store/snapshot_utils.ml index 545a5ac00afe..ee9fb97a480b 100644 --- a/src/lib_layer2_store/snapshot_utils.ml +++ b/src/lib_layer2_store/snapshot_utils.ml @@ -85,7 +85,7 @@ let is_compressed_snapshot snapshot_file = close_in ic ; raise e -module Make (Snapshot_metadata : sig +module Make (Header : sig type t val encoding : t Data_encoding.t @@ -93,18 +93,18 @@ module Make (Snapshot_metadata : sig val size : int end) = struct - let write_snapshot_metadata (module Writer : WRITER_OUTPUT) metadata = - let metadata_bytes = - Data_encoding.Binary.to_bytes_exn Snapshot_metadata.encoding metadata + let write_snapshot_header (module Writer : WRITER_OUTPUT) header = + let header_bytes = + Data_encoding.Binary.to_bytes_exn Header.encoding header in - Writer.output Writer.out_chan metadata_bytes 0 (Bytes.length metadata_bytes) + Writer.output Writer.out_chan header_bytes 0 (Bytes.length header_bytes) - let read_snapshot_metadata (module Reader : READER_INPUT) = - let metadata_bytes = Bytes.create Snapshot_metadata.size in - Reader.really_input Reader.in_chan metadata_bytes 0 Snapshot_metadata.size ; - Data_encoding.Binary.of_bytes_exn Snapshot_metadata.encoding metadata_bytes + let read_snapshot_header (module Reader : READER_INPUT) = + let header_bytes = Bytes.create Header.size in + Reader.really_input Reader.in_chan header_bytes 0 Header.size ; + Data_encoding.Binary.of_bytes_exn Header.encoding header_bytes - let create (module Reader : READER) (module Writer : WRITER) metadata ~dir + let create (module Reader : READER) (module Writer : WRITER) header ~dir ~include_file ~dest = let module Archive_writer = Tar.Make (struct include Reader @@ -169,20 +169,20 @@ struct in let out_chan = Writer.open_out dest in try - write_snapshot_metadata + write_snapshot_header (module struct include Writer let out_chan = out_chan end) - metadata ; + header ; Archive_writer.Archive.create_gen file_stream out_chan ; Writer.close_out out_chan with e -> Writer.close_out out_chan ; raise e - let extract (module Reader : READER) (module Writer : WRITER) metadata_check + let extract (module Reader : READER) (module Writer : WRITER) header_check ~snapshot_file ~dest = let open Lwt_result_syntax in let module Writer = struct @@ -213,13 +213,13 @@ struct in Lwt.finalize (fun () -> - let metadata = read_snapshot_metadata reader_input in - let* check_result = metadata_check metadata in + let header = read_snapshot_header reader_input in + let* check_result = header_check header in let spinner = Progress_bar.spinner ~message:"Extracting snapshot" in Progress_bar.with_reporter spinner @@ fun count_progress -> Writer.count_progress := count_progress ; Archive_reader.Archive.extract_gen out_channel_of_header in_chan ; - return (metadata, check_result)) + return (header, check_result)) (fun () -> Reader.close_in in_chan ; Lwt.return_unit) @@ -256,7 +256,7 @@ struct close_in in_chan ; raise e - let read_metadata (module Reader : READER) ~snapshot_file = + let read_header (module Reader : READER) ~snapshot_file = let in_chan = Reader.open_in snapshot_file in let reader_input : (module READER_INPUT) = (module struct @@ -266,9 +266,9 @@ struct end) in try - let metadata = read_snapshot_metadata reader_input in + let header = read_snapshot_header reader_input in Reader.close_in in_chan ; - metadata + header with e -> Reader.close_in in_chan ; raise e diff --git a/src/lib_layer2_store/snapshot_utils.mli b/src/lib_layer2_store/snapshot_utils.mli index 398f7870a130..ce34d35edffb 100644 --- a/src/lib_layer2_store/snapshot_utils.mli +++ b/src/lib_layer2_store/snapshot_utils.mli @@ -29,38 +29,38 @@ val gzip_writer : writer snapshot file, i.e. a gzip file. *) val is_compressed_snapshot : string -> bool -module Make (Snapshot_metadata : sig +module Make (Header : sig type t val encoding : t Data_encoding.t val size : int end) : sig - (** [create reader writer metadata ~dir ~include_file ~dest] creates a - snapshot archive with the header [metadata] and the hierarchy of files in + (** [create reader writer header ~dir ~include_file ~dest] creates a + snapshot archive with the header [header] and the hierarchy of files in directory [dir] for which [include_file] returns true. The archive is produced in file [dest]. *) val create : reader -> writer -> - Snapshot_metadata.t -> + Header.t -> dir:string -> include_file:(relative_path:string -> bool) -> dest:string -> unit - (** [extract reader writer check_metadata ~snapshot_file ~dest] extracts the + (** [extract reader writer check_header ~snapshot_file ~dest] extracts the snapshot archive [snapshot_file] in the directory [dest]. Existing files - in [dest] with the same names are overwritten. The metadata header read - from the snapshot is checked with [check_metadata] before beginning + in [dest] with the same names are overwritten. The header header read + from the snapshot is checked with [check_header] before beginning extraction, and returned. *) val extract : reader -> writer -> - (Snapshot_metadata.t -> 'a tzresult Lwt.t) -> + (Header.t -> 'a tzresult Lwt.t) -> snapshot_file:string -> dest:string -> - (Snapshot_metadata.t * 'a) tzresult Lwt.t + (Header.t * 'a) tzresult Lwt.t (** [compress ~snapshot_file] compresses the snapshot archive [snapshot_file] of the form ["path/to/snapshot.uncompressed"] to a new file @@ -68,7 +68,7 @@ end) : sig upon successful compression. *) val compress : snapshot_file:string -> string - (** [read_metadata reader ~snapshot_file] reads the metadata from the snapshot + (** [read_header reader ~snapshot_file] reads the header from the snapshot file without extracting it. *) - val read_metadata : reader -> snapshot_file:string -> Snapshot_metadata.t + val read_header : reader -> snapshot_file:string -> Header.t end diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index 59875029ef4d..6c479350f848 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -9,7 +9,7 @@ open Snapshot_utils type compression = No | On_the_fly | After -module Snapshot_metadata = struct +module Header = struct type version = V0 type t = { @@ -25,7 +25,7 @@ module Snapshot_metadata = struct union [ case - ~title:"snapshot_metadata_v0" + ~title:"rollup_node.snapshot.header.v0" (Tag 0) (obj4 (req "history_mode" Configuration.history_mode_encoding) @@ -48,7 +48,7 @@ module Snapshot_metadata = struct |> WithExceptions.Option.get ~loc:__LOC__ end -open Snapshot_utils.Make (Snapshot_metadata) +open Snapshot_utils.Make (Header) let check_store_version store_dir = let open Lwt_result_syntax in @@ -107,7 +107,7 @@ let check_commitment_published cctxt address commitment = in return_unit -let pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir = +let pre_export_checks_and_get_snapshot_header cctxt ~no_checks ~data_dir = let open Lwt_result_syntax in let store_dir = Configuration.default_storage_dir data_dir in let context_dir = Configuration.default_context_dir data_dir in @@ -162,7 +162,7 @@ let pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir = let* () = Store.close store in return { - Snapshot_metadata.version = V0; + Header.version = V0; history_mode; address = metadata.rollup_address; head_level = head.header.level; @@ -419,19 +419,18 @@ let check_l2_chain ~message ~data_dir (store : _ Store.t) context in check_chain head.header.block_hash None -let check_last_commitment head (snapshot_metadata : Snapshot_metadata.t) = +let check_last_commitment head (snapshot_header : Header.t) = let last_snapshot_commitment = Sc_rollup_block.most_recent_commitment head.Sc_rollup_block.header in error_unless - Commitment.Hash.( - snapshot_metadata.last_commitment = last_snapshot_commitment) + Commitment.Hash.(snapshot_header.last_commitment = last_snapshot_commitment) @@ error_of_fmt "Last commitment in snapshot is %a but should be %a." Commitment.Hash.pp last_snapshot_commitment Commitment.Hash.pp - snapshot_metadata.last_commitment + snapshot_header.last_commitment let check_lcc metadata cctxt (store : _ Store.t) (head : Sc_rollup_block.t) (module Plugin : Protocol_plugin_sig.S) = @@ -608,8 +607,8 @@ let maybe_reconstruct_context cctxt ~data_dir ~apply_unsafe_patches = return (Option.is_some head_ctxt)) reconstruct_context_from_first_available_level -let post_checks ?(apply_unsafe_patches = false) ~action ~message - snapshot_metadata ~dest = +let post_checks ?(apply_unsafe_patches = false) ~action ~message snapshot_header + ~dest = let open Lwt_result_syntax in let store_dir = Configuration.default_storage_dir dest in let context_dir = Configuration.default_context_dir dest in @@ -648,7 +647,7 @@ let post_checks ?(apply_unsafe_patches = false) ~action ~message order to verify state hashes. *) failwith "No metadata (needs rollup kind)." | Some metadata -> - let*? () = check_last_commitment head snapshot_metadata in + let*? () = check_last_commitment head snapshot_header in let* () = check_lcc metadata cctxt store head (module Plugin) in return (check_block_data_consistency @@ -670,13 +669,13 @@ let post_export_checks ~snapshot_file = let reader = if is_compressed_snapshot snapshot_file then gzip_reader else stdlib_reader in - let* snapshot_metadata, () = + let* snapshot_header, () = extract reader stdlib_writer (fun _ -> return_unit) ~snapshot_file ~dest in post_checks ~action:`Export ~message:"Checking snapshot " - snapshot_metadata + snapshot_header ~dest let operator_local_file_regexp = Re.Str.regexp "^storage/lpc$" @@ -698,8 +697,8 @@ let with_locks ~data_dir f = ~filename:(Node_context.processing_lockfile_path ~data_dir) f -let export_dir (metadata : Snapshot_metadata.t) ~take_locks ~compression - ~data_dir ~dest ~filename = +let export_dir (header : Header.t) ~take_locks ~compression ~data_dir ~dest + ~filename = let open Lwt_result_syntax in let* snapshot_file = let with_locks = @@ -724,9 +723,9 @@ let export_dir (metadata : Snapshot_metadata.t) ~take_locks ~compression Format.asprintf "snapshot-%a-%ld.%s%s" Address.pp_short - metadata.address - metadata.head_level - (Configuration.string_of_history_mode metadata.history_mode) + header.address + header.head_level + (Configuration.string_of_history_mode header.history_mode) suffix in let dest_file = @@ -749,7 +748,7 @@ let export_dir (metadata : Snapshot_metadata.t) ~take_locks ~compression create stdlib_reader writer - metadata + header ~dir:data_dir ~include_file ~dest:dest_file ; @@ -766,19 +765,25 @@ let export_dir (metadata : Snapshot_metadata.t) ~take_locks ~compression let export cctxt ~no_checks ~compression ~data_dir ~dest ~filename = let open Lwt_result_syntax in - let* metadata = - pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir + let* snapshot_header = + pre_export_checks_and_get_snapshot_header cctxt ~no_checks ~data_dir in let* snapshot_file = - export_dir metadata ~take_locks:true ~compression ~data_dir ~dest ~filename + export_dir + snapshot_header + ~take_locks:true + ~compression + ~data_dir + ~dest + ~filename in let* () = unless no_checks @@ fun () -> post_export_checks ~snapshot_file in return snapshot_file let export_compact cctxt ~no_checks ~compression ~data_dir ~dest ~filename = let open Lwt_result_syntax in - let* snapshot_metadata = - pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir + let* snapshot_header = + pre_export_checks_and_get_snapshot_header cctxt ~no_checks ~data_dir in Lwt_utils_unix.with_tempdir "snapshot_temp_" @@ fun tmp_dir -> let tmp_context_dir = Configuration.default_context_dir tmp_dir in @@ -855,15 +860,14 @@ let export_compact cctxt ~no_checks ~compression ~data_dir ~dest ~filename = | _ -> compression in export_dir - snapshot_metadata + snapshot_header ~take_locks:false ~compression ~data_dir:tmp_dir ~dest ~filename -let pre_import_checks cctxt ~no_checks ~data_dir - (snapshot_metadata : Snapshot_metadata.t) = +let pre_import_checks cctxt ~no_checks ~data_dir (snapshot_header : Header.t) = let open Lwt_result_syntax in let store_dir = Configuration.default_storage_dir data_dir in (* Load stores in read-only to make simple checks. *) @@ -887,26 +891,26 @@ let pre_import_checks cctxt ~no_checks ~data_dir return_unit | Some {rollup_address; _}, Some history_mode -> ( let* () = - error_unless Address.(rollup_address = snapshot_metadata.address) + error_unless Address.(rollup_address = snapshot_header.address) @@ error_of_fmt "The existing rollup node is for %a, but the snapshot is for \ rollup %a." Address.pp rollup_address Address.pp - snapshot_metadata.address + snapshot_header.address in let a_history_str = function | Configuration.Archive -> "an archive" | Configuration.Full -> "a full" in - match (history_mode, snapshot_metadata.history_mode) with + match (history_mode, snapshot_header.history_mode) with | Full, Archive -> Ok () | _, _ -> - error_unless (history_mode = snapshot_metadata.history_mode) + error_unless (history_mode = snapshot_header.history_mode) @@ error_of_fmt "Cannot import %s snapshot into %s rollup node." - (a_history_str snapshot_metadata.history_mode) + (a_history_str snapshot_header.history_mode) (a_history_str history_mode)) in let*? () = @@ -916,19 +920,19 @@ let pre_import_checks cctxt ~no_checks ~data_dir (* The rollup node has no L2 chain. *) return_unit | Some head -> - error_when (snapshot_metadata.head_level <= head.header.level) + error_when (snapshot_header.head_level <= head.header.level) @@ error_of_fmt "The rollup node is already at level %ld but the snapshot is only \ for level %ld." head.header.level - snapshot_metadata.head_level + snapshot_header.head_level in let* () = unless no_checks @@ fun () -> check_commitment_published cctxt - snapshot_metadata.address - snapshot_metadata.last_commitment + snapshot_header.address + snapshot_header.last_commitment in return (metadata, history_mode) @@ -945,11 +949,10 @@ let check_data_dir_unpopulated data_dir () = data_dir else return_unit -let maybe_gc_after_import cctxt ~data_dir - (snapshot_metadata : Snapshot_metadata.t) +let maybe_gc_after_import cctxt ~data_dir (snapshot_header : Header.t) (original_history_mode : Configuration.history_mode option) = let open Lwt_result_syntax in - match (original_history_mode, snapshot_metadata.history_mode) with + match (original_history_mode, snapshot_header.history_mode) with | None, _ -> return_unit | Some Archive, Full -> (* Impossible because filtered out by pre_import_checks. *) @@ -995,7 +998,7 @@ let import ~apply_unsafe_patches ~no_checks ~force cctxt ~data_dir let reader = if is_compressed_snapshot snapshot_file then gzip_reader else stdlib_reader in - let* snapshot_metadata, (_original_metadata, original_history_mode) = + let* snapshot_header, (_original_metadata, original_history_mode) = extract reader stdlib_writer @@ -1005,22 +1008,18 @@ let import ~apply_unsafe_patches ~no_checks ~force cctxt ~data_dir in let* () = maybe_reconstruct_context cctxt ~data_dir ~apply_unsafe_patches in let* () = - maybe_gc_after_import - cctxt - ~data_dir - snapshot_metadata - original_history_mode + maybe_gc_after_import cctxt ~data_dir snapshot_header original_history_mode in unless no_checks @@ fun () -> post_checks ~apply_unsafe_patches ~action:(`Import cctxt) ~message:"Checking imported data" - snapshot_metadata + snapshot_header ~dest:data_dir let info ~snapshot_file = let compressed = is_compressed_snapshot snapshot_file in let reader = if compressed then gzip_reader else stdlib_reader in - let metadata = read_metadata reader ~snapshot_file in - (metadata, if compressed then `Compressed else `Uncompressed) + let snapshot_header = read_header reader ~snapshot_file in + (snapshot_header, if compressed then `Compressed else `Uncompressed) diff --git a/src/lib_smart_rollup_node/snapshots.mli b/src/lib_smart_rollup_node/snapshots.mli index 22015a8ba7e6..e82e932a535d 100644 --- a/src/lib_smart_rollup_node/snapshots.mli +++ b/src/lib_smart_rollup_node/snapshots.mli @@ -17,7 +17,7 @@ type compression = temporarily than {!On_the_fly} but does not lock the rollup node for very long. *) -module Snapshot_metadata : sig +module Header : sig (** Versioning of snapshot format. Only one version for now. *) type version = V0 @@ -85,8 +85,7 @@ val import : (** [info ~snapshot_file] returns information that can be used to inspect the snapshot file. *) -val info : - snapshot_file:string -> Snapshot_metadata.t * [`Compressed | `Uncompressed] +val info : snapshot_file:string -> Header.t * [`Compressed | `Uncompressed] (** [with_modify_data_dir cctxt ~data_dir ~apply_unsafe_patches ?skip_condition f] applies [f] in a read-write context that is created from [data-dir] (and -- GitLab From e219b55cd8d3e3f87c3d4ea9720487611abf8ebd Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 24 Jul 2024 15:28:07 +0200 Subject: [PATCH 7/7] Snapshots: parameterizable file names in archive --- src/lib_layer2_store/snapshot_utils.ml | 19 ++++++------------- src/lib_layer2_store/snapshot_utils.mli | 12 ++++++------ src/lib_smart_rollup_node/snapshots.ml | 20 ++++++++++++-------- 3 files changed, 24 insertions(+), 27 deletions(-) diff --git a/src/lib_layer2_store/snapshot_utils.ml b/src/lib_layer2_store/snapshot_utils.ml index ee9fb97a480b..6918c8582909 100644 --- a/src/lib_layer2_store/snapshot_utils.ml +++ b/src/lib_layer2_store/snapshot_utils.ml @@ -104,22 +104,16 @@ struct Reader.really_input Reader.in_chan header_bytes 0 Header.size ; Data_encoding.Binary.of_bytes_exn Header.encoding header_bytes - let create (module Reader : READER) (module Writer : WRITER) header ~dir - ~include_file ~dest = + let create (module Reader : READER) (module Writer : WRITER) header ~files + ~dest = let module Archive_writer = Tar.Make (struct include Reader include Writer end) in - let files = - Tezos_stdlib_unix.Utils.list_files dir - |> List.filter (fun relative_path -> include_file ~relative_path) - in let total = List.fold_left - (fun total relative_path -> - let {Unix.st_size; _} = - Unix.lstat (Filename.concat dir relative_path) - in + (fun total (file, _) -> + let {Unix.st_size; _} = Unix.lstat file in total + st_size) 0 files @@ -152,14 +146,13 @@ struct in let file_stream = List.rev_map - (fun relative_path -> - let full_path = Filename.concat dir relative_path in + (fun (full_path, path_in_snapshot) -> let {Unix.st_perm; st_size; st_mtime; _} = Unix.lstat full_path in let header = Tar.Header.make ~file_mode:st_perm ~mod_time:(Int64.of_float st_mtime) - relative_path + path_in_snapshot (Int64.of_int st_size) in let writer = write_file full_path in diff --git a/src/lib_layer2_store/snapshot_utils.mli b/src/lib_layer2_store/snapshot_utils.mli index ce34d35edffb..56cf0ef622e4 100644 --- a/src/lib_layer2_store/snapshot_utils.mli +++ b/src/lib_layer2_store/snapshot_utils.mli @@ -36,16 +36,16 @@ module Make (Header : sig val size : int end) : sig - (** [create reader writer header ~dir ~include_file ~dest] creates a - snapshot archive with the header [header] and the hierarchy of files in - directory [dir] for which [include_file] returns true. The archive is - produced in file [dest]. *) + (** [create reader writer header ~files ~dest] creates a snapshot archive with + the header [header] with the contents of [files]. Each element of [files] + is a pair whose first component is the path of the file to include and the + second component is the "relative" path it should be registered to in the + snapshot archive The archive is produced in file [dest]. *) val create : reader -> writer -> Header.t -> - dir:string -> - include_file:(relative_path:string -> bool) -> + files:(string * string) list -> dest:string -> unit diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index 6c479350f848..ec9c57bdae76 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -736,22 +736,26 @@ let export_dir (header : Header.t) ~take_locks ~compression ~data_dir ~dest let*! () = let open Lwt_syntax in let* () = Option.iter_s Lwt_utils_unix.create_dir dest in - let include_file ~relative_path = + let include_file relative_path = Re.Str.string_match snapshotable_files_regexp relative_path 0 && not (Re.Str.string_match operator_local_file_regexp relative_path 0) in + let files = + Tezos_stdlib_unix.Utils.fold_files + data_dir + (fun relative_path acc -> + if not (include_file relative_path) then acc + else + let full_path = Filename.concat data_dir relative_path in + (full_path, relative_path) :: acc) + [] + in let writer = match compression with | On_the_fly -> gzip_writer | No | After -> stdlib_writer in - create - stdlib_reader - writer - header - ~dir:data_dir - ~include_file - ~dest:dest_file ; + create stdlib_reader writer header ~files ~dest:dest_file ; return_unit in return dest_file -- GitLab