diff --git a/tezt/lib_tezos/dal_node.ml b/tezt/lib_tezos/dal_node.ml index 59c53ed8663c453c3202614ff89da8e33badb824..90002162f1dbe45d7bd1e2e6a8d9fd6d7b413297 100644 --- a/tezt/lib_tezos/dal_node.ml +++ b/tezt/lib_tezos/dal_node.ml @@ -78,7 +78,8 @@ let data_dir dal_node = dal_node.persistent_state.data_dir let spawn_command dal_node = Process.spawn ~name:dal_node.name ~color:dal_node.color dal_node.path -let spawn_config_init ?(expected_pow = 0.) ?(peers = []) dal_node = +let spawn_config_init ?(expected_pow = 0.) ?(peers = []) + ?(attestor_profiles = []) ?(producer_profiles = []) dal_node = spawn_command dal_node @@ List.filter_map Fun.id @@ -96,6 +97,10 @@ let spawn_config_init ?(expected_pow = 0.) ?(peers = []) dal_node = Some "--peers"; Some (String.concat "," peers); ] + @ List.concat_map (fun pkh -> ["--attestor-profile"; pkh]) attestor_profiles + @ List.concat_map + (fun slot_index -> ["--producer-profile"; string_of_int slot_index]) + producer_profiles module Config_file = struct let filename dal_node = sf "%s/config.json" @@ data_dir dal_node @@ -107,8 +112,16 @@ module Config_file = struct let update dal_node update = read dal_node |> update |> write dal_node end -let init_config ?expected_pow ?peers dal_node = - let process = spawn_config_init ?expected_pow ?peers dal_node in +let init_config ?expected_pow ?peers ?attestor_profiles ?producer_profiles + dal_node = + let process = + spawn_config_init + ?expected_pow + ?peers + ?attestor_profiles + ?producer_profiles + dal_node + in Process.check process let read_identity dal_node = diff --git a/tezt/lib_tezos/dal_node.mli b/tezt/lib_tezos/dal_node.mli index e7cbbe3321de2ae6205191f6742f1111f1e50b8e..3decca81dcb6f565b65863d8d003b4e7b2a89dc5 100644 --- a/tezt/lib_tezos/dal_node.mli +++ b/tezt/lib_tezos/dal_node.mli @@ -96,7 +96,13 @@ val wait : t -> Unix.process_status Lwt.t [expected_pow] allows to change the PoW difficulty. Default value is 0. *) -val init_config : ?expected_pow:float -> ?peers:string list -> t -> unit Lwt.t +val init_config : + ?expected_pow:float -> + ?peers:string list -> + ?attestor_profiles:string list -> + ?producer_profiles:int list -> + t -> + unit Lwt.t module Config_file : sig (** DAL node configuration files. *) diff --git a/tezt/long_tests/dal.ml b/tezt/long_tests/dal.ml new file mode 100644 index 0000000000000000000000000000000000000000..4017f7b103733fa0ba0371835b751ccbf644326f --- /dev/null +++ b/tezt/long_tests/dal.ml @@ -0,0 +1,296 @@ +(*****************************************************************************) +(* *) +(* Open Source License *) +(* Copyright (c) 2023 Marigold *) +(* *) +(* Permission is hereby granted, free of charge, to any person obtaining a *) +(* copy of this software and associated documentation files (the "Software"),*) +(* to deal in the Software without restriction, including without limitation *) +(* the rights to use, copy, modify, merge, publish, distribute, sublicense, *) +(* and/or sell copies of the Software, and to permit persons to whom the *) +(* Software is furnished to do so, subject to the following conditions: *) +(* *) +(* The above copyright notice and this permission notice shall be included *) +(* in all copies or substantial portions of the Software. *) +(* *) +(* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR*) +(* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, *) +(* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL *) +(* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER*) +(* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING *) +(* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER *) +(* DEALINGS IN THE SOFTWARE. *) +(* *) +(*****************************************************************************) + +(* Testing + ------- + Component: Dal + Invocation: dune exec tezt/long_tests/main.exe -- --file dal.ml + Subject: Performance regression tests for the DAL. +*) + +(* TODO: https://gitlab.com/tezos/tezos/-/issues/6002 + + Add more fine-grained measurements (e.g. time to verify shard) *) + +let measurement = "time-to-produce-and-propagate-shards" + +let grafana_panels : Grafana.panel list = + [ + Row "DAL: Time to produce and propagate shards"; + Grafana.simple_graph ~measurement ~test:measurement ~field:"duration" (); + ] + +(* Adapted from tezt/tests/dal.ml *) +let start_l1_node ~protocol ~account ?l1_bootstrap_peer ?dal_bootstrap_peer () = + let* parameter_file = + let base = Either.right (protocol, Some Protocol.Constants_mainnet) in + let parameter_overrides = + [(["dal_parametric"; "feature_enable"], `Bool true)] + in + Protocol.write_parameter_file + ~base + ~bootstrap_accounts: + (* [bootstrap1] will act as the sole baker + so we include it as [bootstrap_accounts]. *) + [(Constant.bootstrap1, None)] + ~additional_bootstrap_accounts: + (* [bootstrap2] will act as the slot producer + so we provide some tez for posting commitments. *) + [(Constant.bootstrap2, Some 1000000, false)] + parameter_overrides + in + (* Start node with the given protocol *) + let nodes_args = + Node. + [ + Synchronisation_threshold 0; History_mode (Full None); No_bootstrap_peers; + ] + in + let* node, client = + Client.init_with_protocol + ~nodes_args + ~protocol + ~parameter_file + ~keys:[Constant.activator; account] + `Client + () + in + (* Connect the node to the given L1 bootstrap peer (if any). *) + let* () = + match l1_bootstrap_peer with + | None -> unit + | Some peer -> Client.Admin.connect_address ~peer client + in + (* Update [dal_config] in the node config. *) + let* dal_parameters = Rollup.Dal.Parameters.from_client client in + let config : Rollup.Dal.Cryptobox.Config.t = + { + activated = true; + use_mock_srs_for_testing = Some dal_parameters.cryptobox; + bootstrap_peers = + (match dal_bootstrap_peer with + | None -> [] + | Some peer -> [Dal_node.listen_addr peer]); + } + in + Node.Config_file.update + node + (Node.Config_file.set_sandbox_network_with_dal_config config) ; + (* Restart the node to load the new config. *) + let* () = Node.terminate node in + let* () = Node.run node nodes_args in + let* () = Node.wait_for_ready node in + return (node, client) + +let start_dal_node l1_node l1_client ?(producer_profiles = []) + ?(attestor_profiles = []) () = + let dal_node = Dal_node.create ~node:l1_node ~client:l1_client () in + let* _dir = + Dal_node.init_config dal_node ~producer_profiles ~attestor_profiles + in + let* () = Dal_node.run dal_node ~wait_ready:true in + return dal_node + +let store_slot_to_dal_node ~slot_size dal_node = + let slot = Rollup.Dal.make_slot "someslot" ~slot_size in + (* Post a commitment of the slot. *) + let* commitment = RPC.call dal_node (Rollup.Dal.RPC.post_commitment slot) in + (* Compute and save the shards of the slot. *) + let* () = + RPC.call dal_node + @@ Rollup.Dal.RPC.put_commitment_shards ~with_proof:true commitment + in + let commitment_hash = + match Rollup.Dal.Cryptobox.Commitment.of_b58check_opt commitment with + | None -> Test.fail ~__LOC__ "Decoding commitment failed." + | Some hash -> hash + in + (* Compute the proof for the commitment. *) + let* proof = + let* proof = + RPC.call dal_node @@ Rollup.Dal.RPC.get_commitment_proof commitment + in + return + (Data_encoding.Json.destruct + Rollup.Dal.Cryptobox.Commitment_proof.encoding + (`String proof)) + in + return (commitment_hash, proof) + +let publish_slot_header_to_l1_node ~slot_index ~source ~commitment_hash ~proof + client = + let* _op_hash = + Operation.Manager.( + inject + [ + make ~source + @@ dal_publish_slot_header + ~index:slot_index + ~commitment:commitment_hash + ~proof; + ] + client) + in + unit + +let measure f = + let start = Unix.gettimeofday () in + let* res = f () in + return (res, Unix.gettimeofday () -. start) + +let measure_and_add_data_point f ~tag = + let* res, time = measure f in + let data_point = + InfluxDB.data_point ~tags:[("tag", tag)] measurement ("duration", Float time) + in + Long_test.add_data_point data_point ; + return res + +(** Test the time it takes for one node to publish a slot + and another node to download it. + - The sole baker of the network ([node1]) will: + - Bake several blocks so the attestation lag passes. + - Download all the shards from the slot to attest to the availability + of the slot. + - The slot producer ([node2]) will: + - Store the shards for the slot. + - Post a commitment to the slot to L1. +*) +let test_produce_and_propagate_shards ~executors ~protocol = + let repeat = Cli.get_int ~default:10 "repeat" in + let expected_running_time = 120 in + let timeout = Long_test.Seconds (repeat * 10 * expected_running_time) in + Long_test.register + ~__FILE__ + ~title:"DAL node produce and propagate shards" + ~tags:["dal"] + ~executors + ~timeout + @@ fun () -> + Long_test.measure_and_check_regression_lwt + ~tags:[("tag", "total")] + ~repeat + measurement + @@ fun () -> + let slot_index = 0 in + Log.info "Set up [node1], the only baker in the network." ; + let* node1, client1 = + start_l1_node ~protocol ~account:Constant.bootstrap1 () + in + let* dal_node1 = + start_dal_node + node1 + client1 + ~attestor_profiles:[Constant.bootstrap1.public_key_hash] + () + in + Log.info "Set up [node2], the slot producer." ; + let* node2, client2 = + start_l1_node + ~protocol + ~account:Constant.bootstrap2 + ~l1_bootstrap_peer:node1 + ~dal_bootstrap_peer:dal_node1 + () + in + let* dal_node2 = + start_dal_node node2 client2 ~producer_profiles:[slot_index] () + in + let*! () = Client.reveal ~src:Constant.bootstrap2.alias client2 in + let* () = Client.bake_for_and_wait client1 in + Log.info + "Now that the setup of the nodes are finished, we start measuring the time." ; + let* (), time = + measure @@ fun () -> + Log.info "Store slot in [dal_node2]." ; + let* dal_parameters = Rollup.Dal.Parameters.from_client client2 in + let* commitment_hash, proof = + measure_and_add_data_point ~tag:"locally_store_slot" @@ fun () -> + store_slot_to_dal_node + ~slot_size:dal_parameters.cryptobox.slot_size + dal_node2 + in + Log.info "Publish slot header from [node2]." ; + let* () = + publish_slot_header_to_l1_node + ~slot_index + ~source:Constant.bootstrap2 + ~commitment_hash + ~proof + client2 + in + let dal_node_endpoint = Rollup.Dal.endpoint dal_node2 in + Log.info "Bake two blocks to finalize the slot header." ; + let* () = + Base.repeat 2 (fun () -> + Client.bake_for_and_wait client1 ~dal_node_endpoint) + in + Log.info + "Wait until node1 downloads [dal_parameters.cryptobox.number_of_shards] \ + shards. (i.e. the published slot is attestable by node1.)" ; + let* () = + measure_and_add_data_point ~tag:"wait_slot_propagated" @@ fun () -> + let number_of_shards = dal_parameters.cryptobox.number_of_shards in + let shard_count = ref 0 in + Log.info "Waiting to download %d shards..." number_of_shards ; + Dal_node.wait_for dal_node1 "stored_slot_shards.v0" @@ fun e -> + let shards_stored = JSON.(e |-> "shards" |> as_int) in + shard_count := !shard_count + shards_stored ; + if !shard_count = number_of_shards then ( + Log.info "Downloaded %d shards. Ready to attest." number_of_shards ; + Some ()) + else None + in + Log.info "Bake several blocks to surpass the attestation lag." ; + let* () = + Base.repeat (dal_parameters.attestation_lag - 1) (fun () -> + Client.bake_for_and_wait client1 ~dal_node_endpoint) + in + Log.info "Assert that the attestation was indeed posted." ; + let* {dal_attestation; _} = + RPC.(call node1 @@ get_chain_block_metadata ()) + in + Check.((Some [|true|] = dal_attestation) (option (array bool))) + ~error_msg:"Unexpected DAL attestations: expected %L, got %R" ; + unit + in + Log.info + "Terminate all nodes so they don't continue running during the remaining \ + 'repeats'" ; + let* () = + Lwt.join + [ + Node.terminate node1; + Dal_node.terminate dal_node1; + Node.terminate node2; + Dal_node.terminate dal_node2; + ] + in + return time + +let register ~executors ~protocols = + protocols + |> List.iter @@ fun protocol -> + test_produce_and_propagate_shards ~executors ~protocol diff --git a/tezt/long_tests/main.ml b/tezt/long_tests/main.ml index 2dc482ed4e863cb9b707701b32e49e44bf93e881..24bd07ce1ab72937ff7e6e9bdb23534577c825b7 100644 --- a/tezt/long_tests/main.ml +++ b/tezt/long_tests/main.ml @@ -50,7 +50,8 @@ let () = panels = Prt_client.grafana_panels @ Block_validation.grafana_panels @ Tenderbake.grafana_panels @ Logging.grafana_panels - @ Pipelining.grafana_panels @ Tezt_load_time.grafana_panels; + @ Pipelining.grafana_panels @ Tezt_load_time.grafana_panels + @ Dal.grafana_panels; } (* Executor for tests that don't take that long to run. @@ -64,6 +65,7 @@ let () = Prt_client.register ~executors:default_executors ~protocols:[Alpha] ; Sc_rollup.register ~executors:default_executors ~protocols:[Alpha] ; Script_cache.register ~executors:default_executors ~protocols:[Alpha] ; + Dal.register ~executors:default_executors ~protocols:[Alpha] ; Block_validation.register ~executors:default_executors () ; Block_validation.register_semantic_regression_test ~executors:[Long_test.block_replay_executor]