diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index b1adcae5114a6142f6113d102d31604f59a4a46f..53cabe9bb4cfe7db8d4aa5a02e505595580b9dab 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -4407,6 +4407,9 @@ let octez_layer2_store = octez_context_sigs; octez_context_helpers; octez_riscv_pvm; + camlzip; + tar; + tar_unix; ] ~linkall:true ~conflicts:[Conflicts.checkseum] @@ -4669,9 +4672,6 @@ let octez_smart_rollup_node_lib = octez_openapi; octez_node_config; prometheus_app; - camlzip; - tar; - tar_unix; octez_dal_node_lib |> open_; octez_dac_lib |> open_; octez_dac_client_lib |> open_; diff --git a/opam/octez-l2-libs.opam b/opam/octez-l2-libs.opam index 31689d673a678875ceb31980a62833d8c6246b0e..80f0303e3298acf22cf4a33db77f1ee124e63cf8 100644 --- a/opam/octez-l2-libs.opam +++ b/opam/octez-l2-libs.opam @@ -21,6 +21,9 @@ depends: [ "octez-internal-libs" { = version } "aches-lwt" { >= "1.0.0" } "octez-riscv-pvm" { = version } + "camlzip" { >= "1.11" & < "1.12" } + "tar" + "tar-unix" { >= "2.0.1" & < "3.0.0" } "yaml" { >= "3.1.0" } "ppx_import" "qcheck-alcotest" { >= "0.20" } diff --git a/opam/octez-smart-rollup-node-lib.opam b/opam/octez-smart-rollup-node-lib.opam index 9873049828194e9a9dce07c85852ce04fb28c34e..9ecbd96027935429f88868020a057997c6e89bc8 100644 --- a/opam/octez-smart-rollup-node-lib.opam +++ b/opam/octez-smart-rollup-node-lib.opam @@ -15,9 +15,6 @@ depends: [ "cohttp-lwt-unix" { = "5.3.0" } "tezos-openapi" { = version } "octez-node-config" { = version } - "camlzip" { >= "1.11" & < "1.12" } - "tar" - "tar-unix" { >= "2.0.1" & < "3.0.0" } "tezos-dal-node-lib" { = version } "tezos-dac-lib" { = version } "tezos-dac-client-lib" { = version } diff --git a/src/bin_smart_rollup_node/main_smart_rollup_node.ml b/src/bin_smart_rollup_node/main_smart_rollup_node.ml index 1cc44e9d7995e281429d84423554b7e6b296a160..8349fc0c6a07ec82ca34220fc867f8d2ab378d69 100644 --- a/src/bin_smart_rollup_node/main_smart_rollup_node.ml +++ b/src/bin_smart_rollup_node/main_smart_rollup_node.ml @@ -515,7 +515,8 @@ let snapshot_info = (prefixes ["snapshot"; "info"] @@ Cli.snapshot_file_param @@ stop) (fun () snapshot_file cctxt -> let open Lwt_result_syntax in - let ( {Snapshot_utils.history_mode; address; head_level; last_commitment}, + let ( Snapshots.Header. + {version = _; history_mode; address; head_level; last_commitment}, compressed ) = Snapshots.info ~snapshot_file in diff --git a/src/lib_layer2_store/dune b/src/lib_layer2_store/dune index 7006308d99ac6f869f56227b8ba1f38812fe0a47..e3aabc7b2d2e87185b10d9124b90013d769fdc5c 100644 --- a/src/lib_layer2_store/dune +++ b/src/lib_layer2_store/dune @@ -16,7 +16,10 @@ octez-libs.tezos-context.encoding octez-libs.context.sigs octez-libs.tezos-context.helpers - octez-riscv-pvm) + octez-riscv-pvm + camlzip + tar + tar-unix) (library_flags (:standard -linkall)) (flags (:standard) diff --git a/src/lib_layer2_store/snapshot_utils.ml b/src/lib_layer2_store/snapshot_utils.ml new file mode 100644 index 0000000000000000000000000000000000000000..6918c8582909766a4de4312629eb0d9968faebeb --- /dev/null +++ b/src/lib_layer2_store/snapshot_utils.ml @@ -0,0 +1,268 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2023-2024 Functori *) +(* *) +(*****************************************************************************) + +module type READER = sig + type in_channel + + val open_in : string -> in_channel + + val really_input : in_channel -> bytes -> int -> int -> unit + + val input : in_channel -> bytes -> int -> int -> int + + val close_in : in_channel -> unit +end + +module type WRITER = sig + type out_channel + + val open_out : string -> out_channel + + val output : out_channel -> bytes -> int -> int -> unit + + val flush_continue : out_channel -> unit + + val close_out : out_channel -> unit +end + +module type READER_INPUT = sig + include READER + + val in_chan : in_channel +end + +module type WRITER_OUTPUT = sig + include WRITER + + val out_chan : out_channel +end + +module Stdlib_reader : READER with type in_channel = Stdlib.in_channel = Stdlib + +module Stdlib_writer : WRITER with type out_channel = Stdlib.out_channel = +struct + include Stdlib + + let flush_continue = flush +end + +module Gzip_reader : READER with type in_channel = Gzip.in_channel = Gzip + +module Gzip_writer : WRITER with type out_channel = Gzip.out_channel = struct + include Gzip + + let open_out f = open_out f +end + +type reader = (module READER) + +type writer = (module WRITER) + +let stdlib_reader : reader = (module Stdlib_reader) + +let stdlib_writer : writer = (module Stdlib_writer) + +let gzip_reader : reader = (module Gzip_reader) + +let gzip_writer : writer = (module Gzip_writer) + +(* Magic bytes for gzip files is 1f8b. *) +let is_compressed_snapshot snapshot_file = + let ic = open_in snapshot_file in + try + let ok = input_byte ic = 0x1f && input_byte ic = 0x8b in + close_in ic ; + ok + with + | End_of_file -> + close_in ic ; + false + | e -> + close_in ic ; + raise e + +module Make (Header : sig + type t + + val encoding : t Data_encoding.t + + val size : int +end) = +struct + let write_snapshot_header (module Writer : WRITER_OUTPUT) header = + let header_bytes = + Data_encoding.Binary.to_bytes_exn Header.encoding header + in + Writer.output Writer.out_chan header_bytes 0 (Bytes.length header_bytes) + + let read_snapshot_header (module Reader : READER_INPUT) = + let header_bytes = Bytes.create Header.size in + Reader.really_input Reader.in_chan header_bytes 0 Header.size ; + Data_encoding.Binary.of_bytes_exn Header.encoding header_bytes + + let create (module Reader : READER) (module Writer : WRITER) header ~files + ~dest = + let module Archive_writer = Tar.Make (struct + include Reader + include Writer + end) in + let total = + List.fold_left + (fun total (file, _) -> + let {Unix.st_size; _} = Unix.lstat file in + total + st_size) + 0 + files + in + let progress_bar = + Progress_bar.progress_bar + ~counter:`Bytes + ~message:"Exporting snapshot " + ~color:(Terminal.Color.rgb 3 132 252) + total + in + Progress_bar.with_reporter progress_bar @@ fun count_progress -> + let write_file file (out_chan : Writer.out_channel) = + let in_chan = Reader.open_in file in + try + let buffer_size = 64 * 1024 in + let buf = Bytes.create buffer_size in + let rec copy () = + let read_bytes = Reader.input in_chan buf 0 buffer_size in + Writer.output out_chan buf 0 read_bytes ; + count_progress read_bytes ; + if read_bytes > 0 then copy () + in + copy () ; + Writer.flush_continue out_chan ; + Reader.close_in in_chan + with e -> + Reader.close_in in_chan ; + raise e + in + let file_stream = + List.rev_map + (fun (full_path, path_in_snapshot) -> + let {Unix.st_perm; st_size; st_mtime; _} = Unix.lstat full_path in + let header = + Tar.Header.make + ~file_mode:st_perm + ~mod_time:(Int64.of_float st_mtime) + path_in_snapshot + (Int64.of_int st_size) + in + let writer = write_file full_path in + (header, writer)) + files + |> Stream.of_list + in + let out_chan = Writer.open_out dest in + try + write_snapshot_header + (module struct + include Writer + + let out_chan = out_chan + end) + header ; + Archive_writer.Archive.create_gen file_stream out_chan ; + Writer.close_out out_chan + with e -> + Writer.close_out out_chan ; + raise e + + let extract (module Reader : READER) (module Writer : WRITER) header_check + ~snapshot_file ~dest = + let open Lwt_result_syntax in + let module Writer = struct + include Writer + + let count_progress = ref (fun _ -> ()) + + let output oc b p l = + !count_progress 1 ; + output oc b p l + end in + let module Archive_reader = Tar.Make (struct + include Reader + include Writer + end) in + let out_channel_of_header (header : Tar.Header.t) = + let path = Filename.concat dest header.file_name in + Tezos_stdlib_unix.Utils.create_dir (Filename.dirname path) ; + Writer.open_out path + in + let in_chan = Reader.open_in snapshot_file in + let reader_input : (module READER_INPUT) = + (module struct + include Reader + + let in_chan = in_chan + end) + in + Lwt.finalize + (fun () -> + let header = read_snapshot_header reader_input in + let* check_result = header_check header in + let spinner = Progress_bar.spinner ~message:"Extracting snapshot" in + Progress_bar.with_reporter spinner @@ fun count_progress -> + Writer.count_progress := count_progress ; + Archive_reader.Archive.extract_gen out_channel_of_header in_chan ; + return (header, check_result)) + (fun () -> + Reader.close_in in_chan ; + Lwt.return_unit) + + let compress ~snapshot_file = + let Unix.{st_size = total; _} = Unix.stat snapshot_file in + let progress_bar = + Progress_bar.progress_bar + ~counter:`Bytes + ~message:"Compressing snapshot" + ~color:(Terminal.Color.rgb 3 198 252) + total + in + Progress_bar.with_reporter progress_bar @@ fun count_progress -> + let snapshot_file_gz = Filename.chop_suffix snapshot_file ".uncompressed" in + let in_chan = open_in snapshot_file in + let out_chan = Gzip.open_out snapshot_file_gz in + try + let buffer_size = 64 * 1024 in + let buf = Bytes.create buffer_size in + let rec copy () = + let read_bytes = input in_chan buf 0 buffer_size in + Gzip.output out_chan buf 0 read_bytes ; + count_progress read_bytes ; + if read_bytes > 0 then copy () + in + copy () ; + Gzip.close_out out_chan ; + close_in in_chan ; + Unix.unlink snapshot_file ; + snapshot_file_gz + with e -> + Gzip.close_out out_chan ; + close_in in_chan ; + raise e + + let read_header (module Reader : READER) ~snapshot_file = + let in_chan = Reader.open_in snapshot_file in + let reader_input : (module READER_INPUT) = + (module struct + include Reader + + let in_chan = in_chan + end) + in + try + let header = read_snapshot_header reader_input in + Reader.close_in in_chan ; + header + with e -> + Reader.close_in in_chan ; + raise e +end diff --git a/src/lib_layer2_store/snapshot_utils.mli b/src/lib_layer2_store/snapshot_utils.mli new file mode 100644 index 0000000000000000000000000000000000000000..56cf0ef622e42db5976fb4b964386de01fb164fd --- /dev/null +++ b/src/lib_layer2_store/snapshot_utils.mli @@ -0,0 +1,74 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2023-2024 Functori *) +(* *) +(*****************************************************************************) + +(** {2 Snapshot archives creation and extraction} *) + +(** The type of snapshot archive readers. *) +type reader + +(** The type of snapshot archive writers. *) +type writer + +(** A reader for uncompressed files or snapshot archives. *) +val stdlib_reader : reader + +(** A writer for uncompressed files or snapshot archives. *) +val stdlib_writer : writer + +(** A reader for compressed files or snapshot archives. *) +val gzip_reader : reader + +(** A writer for compressed files or snapshot archives. *) +val gzip_writer : writer + +(** [is_compressed_snapshot f] returns [true] if [f] is the path of a compressed + snapshot file, i.e. a gzip file. *) +val is_compressed_snapshot : string -> bool + +module Make (Header : sig + type t + + val encoding : t Data_encoding.t + + val size : int +end) : sig + (** [create reader writer header ~files ~dest] creates a snapshot archive with + the header [header] with the contents of [files]. Each element of [files] + is a pair whose first component is the path of the file to include and the + second component is the "relative" path it should be registered to in the + snapshot archive The archive is produced in file [dest]. *) + val create : + reader -> + writer -> + Header.t -> + files:(string * string) list -> + dest:string -> + unit + + (** [extract reader writer check_header ~snapshot_file ~dest] extracts the + snapshot archive [snapshot_file] in the directory [dest]. Existing files + in [dest] with the same names are overwritten. The header header read + from the snapshot is checked with [check_header] before beginning + extraction, and returned. *) + val extract : + reader -> + writer -> + (Header.t -> 'a tzresult Lwt.t) -> + snapshot_file:string -> + dest:string -> + (Header.t * 'a) tzresult Lwt.t + + (** [compress ~snapshot_file] compresses the snapshot archive [snapshot_file] + of the form ["path/to/snapshot.uncompressed"] to a new file + ["path/to/snapshot"] whose path is returned. [snapshot_file] is removed + upon successful compression. *) + val compress : snapshot_file:string -> string + + (** [read_header reader ~snapshot_file] reads the header from the snapshot + file without extracting it. *) + val read_header : reader -> snapshot_file:string -> Header.t +end diff --git a/src/lib_smart_rollup_node/dune b/src/lib_smart_rollup_node/dune index f7b757fa02cbbd96acd29544f684a36a1a4d6411..1300e4d266949e7c850f9fd9d83fe6be83ddc996 100644 --- a/src/lib_smart_rollup_node/dune +++ b/src/lib_smart_rollup_node/dune @@ -16,9 +16,6 @@ tezos-openapi octez-node-config octez-libs.prometheus-app - camlzip - tar - tar-unix tezos-dal-node-lib tezos-dac-lib tezos-dac-client-lib diff --git a/src/lib_smart_rollup_node/snapshot_utils.ml b/src/lib_smart_rollup_node/snapshot_utils.ml deleted file mode 100644 index 42d273d8c0f74bc8516569374bf640ac17227ead..0000000000000000000000000000000000000000 --- a/src/lib_smart_rollup_node/snapshot_utils.ml +++ /dev/null @@ -1,297 +0,0 @@ -(*****************************************************************************) -(* *) -(* SPDX-License-Identifier: MIT *) -(* Copyright (c) 2023 Functori *) -(* *) -(*****************************************************************************) - -module type READER = sig - type in_channel - - val open_in : string -> in_channel - - val really_input : in_channel -> bytes -> int -> int -> unit - - val input : in_channel -> bytes -> int -> int -> int - - val close_in : in_channel -> unit -end - -module type WRITER = sig - type out_channel - - val open_out : string -> out_channel - - val output : out_channel -> bytes -> int -> int -> unit - - val flush_continue : out_channel -> unit - - val close_out : out_channel -> unit -end - -module type READER_INPUT = sig - include READER - - val in_chan : in_channel -end - -module type WRITER_OUTPUT = sig - include WRITER - - val out_chan : out_channel -end - -module Stdlib_reader : READER with type in_channel = Stdlib.in_channel = Stdlib - -module Stdlib_writer : WRITER with type out_channel = Stdlib.out_channel = -struct - include Stdlib - - let flush_continue = flush -end - -module Gzip_reader : READER with type in_channel = Gzip.in_channel = Gzip - -module Gzip_writer : WRITER with type out_channel = Gzip.out_channel = struct - include Gzip - - let open_out f = open_out f -end - -type reader = (module READER) - -type writer = (module WRITER) - -let stdlib_reader : reader = (module Stdlib_reader) - -let stdlib_writer : writer = (module Stdlib_writer) - -let gzip_reader : reader = (module Gzip_reader) - -let gzip_writer : writer = (module Gzip_writer) - -type snapshot_version = V0 - -type snapshot_metadata = { - history_mode : Configuration.history_mode; - address : Address.t; - head_level : int32; - last_commitment : Commitment.Hash.t; -} - -let snapshot_version_encoding = - let open Data_encoding in - conv_with_guard - (function V0 -> 0) - (function - | 0 -> Ok V0 | x -> Error ("Invalid snapshot version " ^ string_of_int x)) - int8 - -let snaphsot_metadata_encoding = - let open Data_encoding in - conv - (fun {history_mode; address; head_level; last_commitment} -> - (history_mode, address, head_level, last_commitment)) - (fun (history_mode, address, head_level, last_commitment) -> - {history_mode; address; head_level; last_commitment}) - @@ obj4 - (req "history_mode" Configuration.history_mode_encoding) - (req "address" Address.encoding) - (req "head_level" int32) - (req "last_commitment" Commitment.Hash.encoding) - -let snapshot_metadata_size = - Data_encoding.Binary.fixed_length snaphsot_metadata_encoding - |> WithExceptions.Option.get ~loc:__LOC__ - -let version = V0 - -let write_snapshot_metadata (module Writer : WRITER_OUTPUT) metadata = - let version_bytes = - Data_encoding.Binary.to_bytes_exn snapshot_version_encoding version - in - let metadata_bytes = - Data_encoding.Binary.to_bytes_exn snaphsot_metadata_encoding metadata - in - Writer.output Writer.out_chan version_bytes 0 (Bytes.length version_bytes) ; - Writer.output Writer.out_chan metadata_bytes 0 (Bytes.length metadata_bytes) - -let read_snapshot_metadata (module Reader : READER_INPUT) = - let version_bytes = Bytes.create 1 in - let metadata_bytes = Bytes.create snapshot_metadata_size in - Reader.really_input Reader.in_chan version_bytes 0 1 ; - Reader.really_input Reader.in_chan metadata_bytes 0 snapshot_metadata_size ; - let snapshot_version = - Data_encoding.Binary.of_bytes_exn snapshot_version_encoding version_bytes - in - assert (snapshot_version = version) ; - Data_encoding.Binary.of_bytes_exn snaphsot_metadata_encoding metadata_bytes - -let create (module Reader : READER) (module Writer : WRITER) metadata ~dir - ~include_file ~dest = - let module Archive_writer = Tar.Make (struct - include Reader - include Writer - end) in - let files = - Tezos_stdlib_unix.Utils.list_files dir - |> List.filter (fun relative_path -> include_file ~relative_path) - in - let total = - List.fold_left - (fun total relative_path -> - let {Unix.st_size; _} = - Unix.lstat (Filename.concat dir relative_path) - in - total + st_size) - 0 - files - in - let progress_bar = - Progress_bar.progress_bar - ~counter:`Bytes - ~message:"Exporting snapshot " - ~color:(Terminal.Color.rgb 3 132 252) - total - in - Progress_bar.with_reporter progress_bar @@ fun count_progress -> - let write_file file (out_chan : Writer.out_channel) = - let in_chan = Reader.open_in file in - try - let buffer_size = 64 * 1024 in - let buf = Bytes.create buffer_size in - let rec copy () = - let read_bytes = Reader.input in_chan buf 0 buffer_size in - Writer.output out_chan buf 0 read_bytes ; - count_progress read_bytes ; - if read_bytes > 0 then copy () - in - copy () ; - Writer.flush_continue out_chan ; - Reader.close_in in_chan - with e -> - Reader.close_in in_chan ; - raise e - in - let file_stream = - List.rev_map - (fun relative_path -> - let full_path = Filename.concat dir relative_path in - let {Unix.st_perm; st_size; st_mtime; _} = Unix.lstat full_path in - let header = - Tar.Header.make - ~file_mode:st_perm - ~mod_time:(Int64.of_float st_mtime) - relative_path - (Int64.of_int st_size) - in - let writer = write_file full_path in - (header, writer)) - files - |> Stream.of_list - in - let out_chan = Writer.open_out dest in - try - write_snapshot_metadata - (module struct - include Writer - - let out_chan = out_chan - end) - metadata ; - Archive_writer.Archive.create_gen file_stream out_chan ; - Writer.close_out out_chan - with e -> - Writer.close_out out_chan ; - raise e - -let extract (module Reader : READER) (module Writer : WRITER) metadata_check - ~snapshot_file ~dest = - let open Lwt_result_syntax in - let module Writer = struct - include Writer - - let count_progress = ref (fun _ -> ()) - - let output oc b p l = - !count_progress 1 ; - output oc b p l - end in - let module Archive_reader = Tar.Make (struct - include Reader - include Writer - end) in - let out_channel_of_header (header : Tar.Header.t) = - let path = Filename.concat dest header.file_name in - Tezos_stdlib_unix.Utils.create_dir (Filename.dirname path) ; - Writer.open_out path - in - let in_chan = Reader.open_in snapshot_file in - let reader_input : (module READER_INPUT) = - (module struct - include Reader - - let in_chan = in_chan - end) - in - Lwt.finalize - (fun () -> - let metadata = read_snapshot_metadata reader_input in - let* check_result = metadata_check metadata in - let spinner = Progress_bar.spinner ~message:"Extracting snapshot" in - Progress_bar.with_reporter spinner @@ fun count_progress -> - Writer.count_progress := count_progress ; - Archive_reader.Archive.extract_gen out_channel_of_header in_chan ; - return (metadata, check_result)) - (fun () -> - Reader.close_in in_chan ; - Lwt.return_unit) - -let compress ~snapshot_file = - let Unix.{st_size = total; _} = Unix.stat snapshot_file in - let progress_bar = - Progress_bar.progress_bar - ~counter:`Bytes - ~message:"Compressing snapshot" - ~color:(Terminal.Color.rgb 3 198 252) - total - in - Progress_bar.with_reporter progress_bar @@ fun count_progress -> - let snapshot_file_gz = Filename.chop_suffix snapshot_file ".uncompressed" in - let in_chan = open_in snapshot_file in - let out_chan = Gzip.open_out snapshot_file_gz in - try - let buffer_size = 64 * 1024 in - let buf = Bytes.create buffer_size in - let rec copy () = - let read_bytes = input in_chan buf 0 buffer_size in - Gzip.output out_chan buf 0 read_bytes ; - count_progress read_bytes ; - if read_bytes > 0 then copy () - in - copy () ; - Gzip.close_out out_chan ; - close_in in_chan ; - Unix.unlink snapshot_file ; - snapshot_file_gz - with e -> - Gzip.close_out out_chan ; - close_in in_chan ; - raise e - -let read_metadata (module Reader : READER) ~snapshot_file = - let in_chan = Reader.open_in snapshot_file in - let reader_input : (module READER_INPUT) = - (module struct - include Reader - - let in_chan = in_chan - end) - in - try - let metadata = read_snapshot_metadata reader_input in - Reader.close_in in_chan ; - metadata - with e -> - Reader.close_in in_chan ; - raise e diff --git a/src/lib_smart_rollup_node/snapshot_utils.mli b/src/lib_smart_rollup_node/snapshot_utils.mli deleted file mode 100644 index 79e47d3a4ec0194be49d22b7c7ead50358d82bd0..0000000000000000000000000000000000000000 --- a/src/lib_smart_rollup_node/snapshot_utils.mli +++ /dev/null @@ -1,74 +0,0 @@ -(*****************************************************************************) -(* *) -(* SPDX-License-Identifier: MIT *) -(* Copyright (c) 2023 Functori *) -(* *) -(*****************************************************************************) - -(** {2 Snapshot archives creation and extraction} *) - -(** The type of snapshot archive readers. *) -type reader - -(** The type of snapshot archive writers. *) -type writer - -(** A reader for uncompressed files or snapshot archives. *) -val stdlib_reader : reader - -(** A writer for uncompressed files or snapshot archives. *) -val stdlib_writer : writer - -(** A reader for compressed files or snapshot archives. *) -val gzip_reader : reader - -(** A writer for compressed files or snapshot archives. *) -val gzip_writer : writer - -(** Versioning of snapshot format. Only one version for now. *) -type snapshot_version = V0 - -(** Snapshot metadata for version 0. This information is written as a header of - the archive snapshot file. *) -type snapshot_metadata = { - history_mode : Configuration.history_mode; - address : Address.t; - head_level : int32; - last_commitment : Commitment.Hash.t; -} - -(** [create reader writer metadata ~dir ~include_file ~dest] creates a snapshot - archive with the header [metadata] and the hierarchy of files in directory - [dir] for which [include_file] returns true. The archive is produced in file - [dest]. *) -val create : - reader -> - writer -> - snapshot_metadata -> - dir:string -> - include_file:(relative_path:string -> bool) -> - dest:string -> - unit - -(** [extract reader writer check_metadata ~snapshot_file ~dest] extracts the - snapshot archive [snapshot_file] in the directory [dest]. Existing files in - [dest] with the same names are overwritten. The metadata header read from - the snapshot is checked with [check_metadata] before beginning - extraction, and returned. *) -val extract : - reader -> - writer -> - (snapshot_metadata -> 'a tzresult Lwt.t) -> - snapshot_file:string -> - dest:string -> - (snapshot_metadata * 'a) tzresult Lwt.t - -(** [compress ~snapshot_file] compresses the snapshot archive [snapshot_file] of - the form ["path/to/snapshot.uncompressed"] to a new file - ["path/to/snapshot"] whose path is returned. [snapshot_file] is removed upon - successful compression. *) -val compress : snapshot_file:string -> string - -(** [read_metadata reader ~snapshot_file] reads the metadata from the snapshot - file without extracting it. *) -val read_metadata : reader -> snapshot_file:string -> snapshot_metadata diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index d64363d4c0eb5a947ba395903f58bcfec0b4ebc6..ec9c57bdae76dee5a54e767ee8596c608c8d0755 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -9,6 +9,47 @@ open Snapshot_utils type compression = No | On_the_fly | After +module Header = struct + type version = V0 + + type t = { + version : version; + history_mode : Configuration.history_mode; + address : Address.t; + head_level : int32; + last_commitment : Commitment.Hash.t; + } + + let encoding = + let open Data_encoding in + union + [ + case + ~title:"rollup_node.snapshot.header.v0" + (Tag 0) + (obj4 + (req "history_mode" Configuration.history_mode_encoding) + (req "address" Address.encoding) + (req "head_level" int32) + (req "last_commitment" Commitment.Hash.encoding)) + (fun { + version = V0; + history_mode; + address; + head_level; + last_commitment; + } -> Some (history_mode, address, head_level, last_commitment)) + (fun (history_mode, address, head_level, last_commitment) -> + {version = V0; history_mode; address; head_level; last_commitment}); + ] + + let size = + Data_encoding.Binary.fixed_length encoding + |> WithExceptions.Option.get ~loc:__LOC__ +end + +open Snapshot_utils.Make (Header) + let check_store_version store_dir = let open Lwt_result_syntax in let* store_version = Store_version.read_version_file ~dir:store_dir in @@ -66,7 +107,7 @@ let check_commitment_published cctxt address commitment = in return_unit -let pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir = +let pre_export_checks_and_get_snapshot_header cctxt ~no_checks ~data_dir = let open Lwt_result_syntax in let store_dir = Configuration.default_storage_dir data_dir in let context_dir = Configuration.default_context_dir data_dir in @@ -121,6 +162,7 @@ let pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir = let* () = Store.close store in return { + Header.version = V0; history_mode; address = metadata.rollup_address; head_level = head.header.level; @@ -377,19 +419,18 @@ let check_l2_chain ~message ~data_dir (store : _ Store.t) context in check_chain head.header.block_hash None -let check_last_commitment head snapshot_metadata = +let check_last_commitment head (snapshot_header : Header.t) = let last_snapshot_commitment = Sc_rollup_block.most_recent_commitment head.Sc_rollup_block.header in error_unless - Commitment.Hash.( - snapshot_metadata.last_commitment = last_snapshot_commitment) + Commitment.Hash.(snapshot_header.last_commitment = last_snapshot_commitment) @@ error_of_fmt "Last commitment in snapshot is %a but should be %a." Commitment.Hash.pp last_snapshot_commitment Commitment.Hash.pp - snapshot_metadata.last_commitment + snapshot_header.last_commitment let check_lcc metadata cctxt (store : _ Store.t) (head : Sc_rollup_block.t) (module Plugin : Protocol_plugin_sig.S) = @@ -566,8 +607,8 @@ let maybe_reconstruct_context cctxt ~data_dir ~apply_unsafe_patches = return (Option.is_some head_ctxt)) reconstruct_context_from_first_available_level -let post_checks ?(apply_unsafe_patches = false) ~action ~message - snapshot_metadata ~dest = +let post_checks ?(apply_unsafe_patches = false) ~action ~message snapshot_header + ~dest = let open Lwt_result_syntax in let store_dir = Configuration.default_storage_dir dest in let context_dir = Configuration.default_context_dir dest in @@ -606,7 +647,7 @@ let post_checks ?(apply_unsafe_patches = false) ~action ~message order to verify state hashes. *) failwith "No metadata (needs rollup kind)." | Some metadata -> - let*? () = check_last_commitment head snapshot_metadata in + let*? () = check_last_commitment head snapshot_header in let* () = check_lcc metadata cctxt store head (module Plugin) in return (check_block_data_consistency @@ -622,34 +663,19 @@ let post_checks ?(apply_unsafe_patches = false) ~action ~message let* () = Store.close store in return_unit -(* Magic bytes for gzip files is 1f8b. *) -let is_compressed_snapshot snapshot_file = - let ic = open_in snapshot_file in - try - let ok = input_byte ic = 0x1f && input_byte ic = 0x8b in - close_in ic ; - ok - with - | End_of_file -> - close_in ic ; - false - | e -> - close_in ic ; - raise e - let post_export_checks ~snapshot_file = let open Lwt_result_syntax in Lwt_utils_unix.with_tempdir "snapshot_checks_" @@ fun dest -> let reader = if is_compressed_snapshot snapshot_file then gzip_reader else stdlib_reader in - let* snapshot_metadata, () = + let* snapshot_header, () = extract reader stdlib_writer (fun _ -> return_unit) ~snapshot_file ~dest in post_checks ~action:`Export ~message:"Checking snapshot " - snapshot_metadata + snapshot_header ~dest let operator_local_file_regexp = Re.Str.regexp "^storage/lpc$" @@ -671,7 +697,8 @@ let with_locks ~data_dir f = ~filename:(Node_context.processing_lockfile_path ~data_dir) f -let export_dir metadata ~take_locks ~compression ~data_dir ~dest ~filename = +let export_dir (header : Header.t) ~take_locks ~compression ~data_dir ~dest + ~filename = let open Lwt_result_syntax in let* snapshot_file = let with_locks = @@ -696,9 +723,9 @@ let export_dir metadata ~take_locks ~compression ~data_dir ~dest ~filename = Format.asprintf "snapshot-%a-%ld.%s%s" Address.pp_short - metadata.address - metadata.head_level - (Configuration.string_of_history_mode metadata.history_mode) + header.address + header.head_level + (Configuration.string_of_history_mode header.history_mode) suffix in let dest_file = @@ -709,22 +736,26 @@ let export_dir metadata ~take_locks ~compression ~data_dir ~dest ~filename = let*! () = let open Lwt_syntax in let* () = Option.iter_s Lwt_utils_unix.create_dir dest in - let include_file ~relative_path = + let include_file relative_path = Re.Str.string_match snapshotable_files_regexp relative_path 0 && not (Re.Str.string_match operator_local_file_regexp relative_path 0) in + let files = + Tezos_stdlib_unix.Utils.fold_files + data_dir + (fun relative_path acc -> + if not (include_file relative_path) then acc + else + let full_path = Filename.concat data_dir relative_path in + (full_path, relative_path) :: acc) + [] + in let writer = match compression with | On_the_fly -> gzip_writer | No | After -> stdlib_writer in - create - stdlib_reader - writer - metadata - ~dir:data_dir - ~include_file - ~dest:dest_file ; + create stdlib_reader writer header ~files ~dest:dest_file ; return_unit in return dest_file @@ -738,19 +769,25 @@ let export_dir metadata ~take_locks ~compression ~data_dir ~dest ~filename = let export cctxt ~no_checks ~compression ~data_dir ~dest ~filename = let open Lwt_result_syntax in - let* metadata = - pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir + let* snapshot_header = + pre_export_checks_and_get_snapshot_header cctxt ~no_checks ~data_dir in let* snapshot_file = - export_dir metadata ~take_locks:true ~compression ~data_dir ~dest ~filename + export_dir + snapshot_header + ~take_locks:true + ~compression + ~data_dir + ~dest + ~filename in let* () = unless no_checks @@ fun () -> post_export_checks ~snapshot_file in return snapshot_file let export_compact cctxt ~no_checks ~compression ~data_dir ~dest ~filename = let open Lwt_result_syntax in - let* snapshot_metadata = - pre_export_checks_and_get_snapshot_metadata cctxt ~no_checks ~data_dir + let* snapshot_header = + pre_export_checks_and_get_snapshot_header cctxt ~no_checks ~data_dir in Lwt_utils_unix.with_tempdir "snapshot_temp_" @@ fun tmp_dir -> let tmp_context_dir = Configuration.default_context_dir tmp_dir in @@ -827,14 +864,14 @@ let export_compact cctxt ~no_checks ~compression ~data_dir ~dest ~filename = | _ -> compression in export_dir - snapshot_metadata + snapshot_header ~take_locks:false ~compression ~data_dir:tmp_dir ~dest ~filename -let pre_import_checks cctxt ~no_checks ~data_dir snapshot_metadata = +let pre_import_checks cctxt ~no_checks ~data_dir (snapshot_header : Header.t) = let open Lwt_result_syntax in let store_dir = Configuration.default_storage_dir data_dir in (* Load stores in read-only to make simple checks. *) @@ -858,26 +895,26 @@ let pre_import_checks cctxt ~no_checks ~data_dir snapshot_metadata = return_unit | Some {rollup_address; _}, Some history_mode -> ( let* () = - error_unless Address.(rollup_address = snapshot_metadata.address) + error_unless Address.(rollup_address = snapshot_header.address) @@ error_of_fmt "The existing rollup node is for %a, but the snapshot is for \ rollup %a." Address.pp rollup_address Address.pp - snapshot_metadata.address + snapshot_header.address in let a_history_str = function | Configuration.Archive -> "an archive" | Configuration.Full -> "a full" in - match (history_mode, snapshot_metadata.history_mode) with + match (history_mode, snapshot_header.history_mode) with | Full, Archive -> Ok () | _, _ -> - error_unless (history_mode = snapshot_metadata.history_mode) + error_unless (history_mode = snapshot_header.history_mode) @@ error_of_fmt "Cannot import %s snapshot into %s rollup node." - (a_history_str snapshot_metadata.history_mode) + (a_history_str snapshot_header.history_mode) (a_history_str history_mode)) in let*? () = @@ -887,19 +924,19 @@ let pre_import_checks cctxt ~no_checks ~data_dir snapshot_metadata = (* The rollup node has no L2 chain. *) return_unit | Some head -> - error_when (snapshot_metadata.head_level <= head.header.level) + error_when (snapshot_header.head_level <= head.header.level) @@ error_of_fmt "The rollup node is already at level %ld but the snapshot is only \ for level %ld." head.header.level - snapshot_metadata.head_level + snapshot_header.head_level in let* () = unless no_checks @@ fun () -> check_commitment_published cctxt - snapshot_metadata.address - snapshot_metadata.last_commitment + snapshot_header.address + snapshot_header.last_commitment in return (metadata, history_mode) @@ -916,10 +953,10 @@ let check_data_dir_unpopulated data_dir () = data_dir else return_unit -let maybe_gc_after_import cctxt ~data_dir snapshot_metadata +let maybe_gc_after_import cctxt ~data_dir (snapshot_header : Header.t) (original_history_mode : Configuration.history_mode option) = let open Lwt_result_syntax in - match (original_history_mode, snapshot_metadata.history_mode) with + match (original_history_mode, snapshot_header.history_mode) with | None, _ -> return_unit | Some Archive, Full -> (* Impossible because filtered out by pre_import_checks. *) @@ -965,7 +1002,7 @@ let import ~apply_unsafe_patches ~no_checks ~force cctxt ~data_dir let reader = if is_compressed_snapshot snapshot_file then gzip_reader else stdlib_reader in - let* snapshot_metadata, (_original_metadata, original_history_mode) = + let* snapshot_header, (_original_metadata, original_history_mode) = extract reader stdlib_writer @@ -975,22 +1012,18 @@ let import ~apply_unsafe_patches ~no_checks ~force cctxt ~data_dir in let* () = maybe_reconstruct_context cctxt ~data_dir ~apply_unsafe_patches in let* () = - maybe_gc_after_import - cctxt - ~data_dir - snapshot_metadata - original_history_mode + maybe_gc_after_import cctxt ~data_dir snapshot_header original_history_mode in unless no_checks @@ fun () -> post_checks ~apply_unsafe_patches ~action:(`Import cctxt) ~message:"Checking imported data" - snapshot_metadata + snapshot_header ~dest:data_dir let info ~snapshot_file = let compressed = is_compressed_snapshot snapshot_file in let reader = if compressed then gzip_reader else stdlib_reader in - let metadata = read_metadata reader ~snapshot_file in - (metadata, if compressed then `Compressed else `Uncompressed) + let snapshot_header = read_header reader ~snapshot_file in + (snapshot_header, if compressed then `Compressed else `Uncompressed) diff --git a/src/lib_smart_rollup_node/snapshots.mli b/src/lib_smart_rollup_node/snapshots.mli index 35c47e33794949d14d97bdecde5058027a0aa858..e82e932a535dc0bfcc6c233ed122ecd13eb4f871 100644 --- a/src/lib_smart_rollup_node/snapshots.mli +++ b/src/lib_smart_rollup_node/snapshots.mli @@ -17,6 +17,24 @@ type compression = temporarily than {!On_the_fly} but does not lock the rollup node for very long. *) +module Header : sig + (** Versioning of snapshot format. Only one version for now. *) + type version = V0 + + (** Snapshot metadata for version 0. This information is written as a header of + the archive snapshot file. *) + type t = { + version : version; + history_mode : Configuration.history_mode; + address : Address.t; + head_level : int32; + last_commitment : Commitment.Hash.t; + } + + (** Fixed size metadata encoding. *) + val encoding : t Data_encoding.t +end + (** [export cctxt ~no_checks ~compression ~data_dir ~dest ~filename] creates a tar gzipped archive with name [filename] (or a generated name) in [dest] (or the current directory) containing a snapshot of the data of the rollup node @@ -67,9 +85,7 @@ val import : (** [info ~snapshot_file] returns information that can be used to inspect the snapshot file. *) -val info : - snapshot_file:string -> - Snapshot_utils.snapshot_metadata * [`Compressed | `Uncompressed] +val info : snapshot_file:string -> Header.t * [`Compressed | `Uncompressed] (** [with_modify_data_dir cctxt ~data_dir ~apply_unsafe_patches ?skip_condition f] applies [f] in a read-write context that is created from [data-dir] (and