From e6d223770276e997a609cfee6142005c0a55e7ae Mon Sep 17 00:00:00 2001
From: Romain Bardou <romain@nomadic-labs.com>
Date: Wed, 31 Mar 2021 12:49:05 +0200
Subject: [PATCH 1/9] Tezt: fix shell_quote

In the case where the string contained a single quote,
it did not quote some characters which have meaning for the shell,
such as parentheses.
---
 tezt/lib/log.ml | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/tezt/lib/log.ml b/tezt/lib/log.ml
index d94150b81ef9..b379bd26943a 100644
--- a/tezt/lib/log.ml
+++ b/tezt/lib/log.ml
@@ -55,7 +55,8 @@ let channel =
   in
   match list with None -> stdout | Some (`Ascii_art | `Tsv) -> stderr
 
-(* In theory we could simply escape spaces, backslashes, double quotes and single quotes.
+(* In theory we could simply escape spaces, backslashes, double quotes, single quotes
+   and other symbols with a meaning for the shell.
    But 'some long argument' is arguably more readable than some\ long\ argument.
    We use this quoting method if the string contains no single quote. *)
 let quote_shell s =
@@ -83,16 +84,7 @@ let quote_shell s =
   String.iter categorize s ;
   if not !needs_quotes then s
   else if not !contains_single_quote then "'" ^ s ^ "'"
-  else
-    let buffer = Buffer.create (String.length s * 2) in
-    let add_char = function
-      | (' ' | '\\' | '"' | '\'') as c ->
-          Buffer.add_char buffer '\\' ;
-          Buffer.add_char buffer c
-      | c ->
-          Buffer.add_char buffer c
-    in
-    String.iter add_char s ; Buffer.contents buffer
+  else Filename.quote s
 
 let quote_shell_command command arguments =
   String.concat " " (List.map quote_shell (command :: arguments))
-- 
GitLab


From d493935a3ab0568b690534fa994ad68209b0bf79 Mon Sep 17 00:00:00 2001
From: Romain Bardou <romain@nomadic-labs.com>
Date: Mon, 8 Mar 2021 18:26:08 +0100
Subject: [PATCH 2/9] Tezt: add --not-test

---
 tezt/lib/cli.ml  | 11 ++++++++++-
 tezt/lib/cli.mli |  1 +
 tezt/lib/log.ml  |  1 +
 tezt/lib/test.ml |  1 +
 4 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/tezt/lib/cli.ml b/tezt/lib/cli.ml
index 58c22316c990..2d24d7e53b2d 100644
--- a/tezt/lib/cli.ml
+++ b/tezt/lib/cli.ml
@@ -38,6 +38,7 @@ type options = {
   keep_going : bool;
   files_to_run : string list;
   tests_to_run : string list;
+  tests_not_to_run : string list;
   tags_to_run : string list;
   tags_not_to_run : string list;
   list : [`Ascii_art | `Tsv] option;
@@ -61,6 +62,7 @@ let options =
   let keep_going = ref false in
   let files_to_run = ref [] in
   let tests_to_run = ref [] in
+  let tests_not_to_run = ref [] in
   let tags_to_run = ref [] in
   let tags_not_to_run = ref [] in
   let list = ref None in
@@ -159,6 +161,11 @@ let options =
         ( "-t",
           Arg.String (fun title -> tests_to_run := title :: !tests_to_run),
           "<TITLE> Same as --test." );
+        ( "--not-test",
+          Arg.String
+            (fun title -> tests_not_to_run := title :: !tests_not_to_run),
+          "<TITLE> Only run tests which are not exactly entitled TITLE (see \
+           SELECTING TESTS)." );
         ( "--global-timeout",
           Arg.Float (fun delay -> global_timeout := Some delay),
           "<SECONDS> Fail if the set of tests takes more than SECONDS to run."
@@ -201,7 +208,8 @@ let options =
        negate a tag, prefix it with a slash: /\n\n\
       \  The title of a test is given by the ~title argument of Test.run. It \
        is what is printed after [SUCCESS] (or [FAILURE] or [ABORTED]) in the \
-       reports. Use --test to select a test by its title on the command-line.\n\n\
+       reports. Use --test (respectively --not-test) to select (respectively \
+       unselect) a test by its title on the command-line.\n\n\
       \  The file in which a test is implemented is specified by the \
        ~__FILE__ argument of Test.run. In other words, it is the name of the \
        file in which the test is defined, without directories. Use --file to \
@@ -231,6 +239,7 @@ let options =
     keep_going = !keep_going;
     files_to_run = !files_to_run;
     tests_to_run = !tests_to_run;
+    tests_not_to_run = !tests_not_to_run;
     tags_to_run = !tags_to_run;
     tags_not_to_run = !tags_not_to_run;
     list = !list;
diff --git a/tezt/lib/cli.mli b/tezt/lib/cli.mli
index 231aa859498e..d59f83b22863 100644
--- a/tezt/lib/cli.mli
+++ b/tezt/lib/cli.mli
@@ -69,6 +69,7 @@ type options = {
   keep_going : bool;
   files_to_run : string list;
   tests_to_run : string list;
+  tests_not_to_run : string list;
   tags_to_run : string list;
   tags_not_to_run : string list;
   list : [`Ascii_art | `Tsv] option;
diff --git a/tezt/lib/log.ml b/tezt/lib/log.ml
index b379bd26943a..891b48b303be 100644
--- a/tezt/lib/log.ml
+++ b/tezt/lib/log.ml
@@ -42,6 +42,7 @@ let channel =
         keep_going = _;
         files_to_run = _;
         tests_to_run = _;
+        tests_not_to_run = _;
         tags_to_run = _;
         tags_not_to_run = _;
         list;
diff --git a/tezt/lib/test.ml b/tezt/lib/test.ml
index 3ce55ce18c0d..ae9ecd08daa2 100644
--- a/tezt/lib/test.ml
+++ b/tezt/lib/test.ml
@@ -210,6 +210,7 @@ let test_should_be_run ~file ~title ~tags =
          true
      | titles ->
          List.mem title titles )
+  && (not (List.mem title Cli.options.tests_not_to_run))
   &&
   match Cli.options.files_to_run with
   | [] ->
-- 
GitLab


From 7c0e35bfb7c2047dff57ebe141815d5dece5b748 Mon Sep 17 00:00:00 2001
From: Romain Bardou <romain@nomadic-labs.com>
Date: Mon, 8 Mar 2021 18:21:52 +0100
Subject: [PATCH 3/9] Tezt: add --record, --job-count and --suggest-jobs

---
 tezt/lib/base.mli |  1 -
 tezt/lib/cli.ml   | 31 ++++++++++++++-
 tezt/lib/cli.mli  |  3 ++
 tezt/lib/log.ml   | 11 +++++-
 tezt/lib/test.ml  | 99 +++++++++++++++++++++++++++++++++++++++++++++--
 5 files changed, 137 insertions(+), 8 deletions(-)

diff --git a/tezt/lib/base.mli b/tezt/lib/base.mli
index 7ce05a04caae..749206ac62d3 100644
--- a/tezt/lib/base.mli
+++ b/tezt/lib/base.mli
@@ -118,7 +118,6 @@ val wait_for_async : unit -> unit Lwt.t
 (** Repeat something a given amount of times. *)
 val repeat : int -> (unit -> unit Lwt.t) -> unit Lwt.t
 
-
 (** {2 Input/Output} *)
 
 (** Open file, use function to write output then close the output. In case of
diff --git a/tezt/lib/cli.ml b/tezt/lib/cli.ml
index 2d24d7e53b2d..6f030281e366 100644
--- a/tezt/lib/cli.ml
+++ b/tezt/lib/cli.ml
@@ -48,6 +48,9 @@ type options = {
   loop : bool;
   time : bool;
   starting_port : int;
+  record : string option;
+  job_count : int;
+  suggest_jobs : string option;
 }
 
 let options =
@@ -72,6 +75,9 @@ let options =
   let loop = ref false in
   let time = ref false in
   let starting_port = ref 16384 in
+  let record = ref None in
+  let job_count = ref 3 in
+  let suggest_jobs = ref None in
   let set_log_level = function
     | "quiet" ->
         log_level := Quiet
@@ -88,6 +94,10 @@ let options =
     | level ->
         raise (Arg.Bad (Printf.sprintf "invalid log level: %S" level))
   in
+  let set_job_count value =
+    if value < 1 then raise (Arg.Bad "--job-count must be positive") ;
+    job_count := value
+  in
   let spec =
     Arg.align
       [ ("--color", Arg.Set color, " Use colors in output.");
@@ -191,7 +201,23 @@ let options =
            failed." );
         ( "--starting-port",
           Arg.Set_int starting_port,
-          " If tests need to open ports, they may start from this number." ) ]
+          " If tests need to open ports, they may start from this number." );
+        ( "--record",
+          Arg.String (fun file -> record := Some file),
+          "<FILE> Record test results to FILE. This file can then be used \
+           with --suggest-jobs." );
+        ( "--job-count",
+          Arg.Int set_job_count,
+          "<COUNT> Set the number of target jobs for --suggest-jobs (default \
+           is 3)." );
+        ("-j", Arg.Int set_job_count, "<COUNT> Same as --job-count.");
+        ( "--suggest-jobs",
+          Arg.String (fun file -> suggest_jobs := Some file),
+          "<FILE> Read test results from a file generated with --record and \
+           suggest a partition of the tests that would result in --job-count \
+           sets of roughly the same total duration. Output each job as a list \
+           of flags that can be passed to Tezt, followed by a shell comment \
+           that denotes the expected duration of the job." ) ]
   in
   let usage =
     (* This was formatted by ocamlformat. Sorry for all the slashes. *)
@@ -249,4 +275,7 @@ let options =
     loop = !loop;
     time = !time;
     starting_port = !starting_port;
+    record = !record;
+    job_count = !job_count;
+    suggest_jobs = !suggest_jobs;
   }
diff --git a/tezt/lib/cli.mli b/tezt/lib/cli.mli
index d59f83b22863..2d7781694a6b 100644
--- a/tezt/lib/cli.mli
+++ b/tezt/lib/cli.mli
@@ -79,6 +79,9 @@ type options = {
   loop : bool;
   time : bool;
   starting_port : int;
+  record : string option;
+  job_count : int;
+  suggest_jobs : string option;
 }
 
 (** Values for command-line options. *)
diff --git a/tezt/lib/log.ml b/tezt/lib/log.ml
index 891b48b303be..32900f0bdac6 100644
--- a/tezt/lib/log.ml
+++ b/tezt/lib/log.ml
@@ -51,10 +51,17 @@ let channel =
         reset_regressions = _;
         loop = _;
         time = _;
-        starting_port = _ } =
+        starting_port = _;
+        record = _;
+        job_count = _;
+        suggest_jobs } =
     Cli.options
   in
-  match list with None -> stdout | Some (`Ascii_art | `Tsv) -> stderr
+  if
+    (match list with None -> false | Some (`Ascii_art | `Tsv) -> true)
+    || match suggest_jobs with None -> false | Some _ -> true
+  then stderr
+  else stdout
 
 (* In theory we could simply escape spaces, backslashes, double quotes, single quotes
    and other symbols with a meaning for the shell.
diff --git a/tezt/lib/test.ml b/tezt/lib/test.ml
index ae9ecd08daa2..5424d328d174 100644
--- a/tezt/lib/test.ml
+++ b/tezt/lib/test.ml
@@ -369,6 +369,86 @@ let display_time_summary () =
   String_map.iter print_time_for_file tests_by_file ;
   ()
 
+type marshaled_test = {
+  file : string;
+  title : string;
+  tags : string list;
+  time : float;
+}
+
+let record_results filename =
+  (* Remove the closure ([body]). *)
+  let marshaled_tests =
+    List.map
+      (fun {file; title; tags; body = _; time} -> {file; title; tags; time})
+      !list
+  in
+  (* Write to file using Marshal.
+     This is not very robust but enough for the purposes of this file. *)
+  try
+    with_open_out filename
+    @@ fun file ->
+    Marshal.to_channel file (marshaled_tests : marshaled_test list) []
+  with Sys_error error -> Log.warn "Failed to write record: %s\n%!" error
+
+let read_recorded_results filename : marshaled_test list =
+  with_open_in filename Marshal.from_channel
+
+let suggest_jobs (tests : marshaled_test list) =
+  let job_count = max 1 Cli.options.job_count in
+  (* [jobs] is an array of pairs where the first value is the total time of the job
+     and the second value is the list of tests that are currently allocated to this job. *)
+  let jobs = Array.make job_count (0., []) in
+  let allocate test =
+    let smallest_job =
+      let best_index = ref 0 in
+      let best_time = ref max_float in
+      for i = 0 to job_count - 1 do
+        let (job_time, _) = jobs.(i) in
+        if job_time < !best_time then (
+          best_index := i ;
+          best_time := job_time )
+      done ;
+      !best_index
+    in
+    let (job_time, job_tests) = jobs.(smallest_job) in
+    jobs.(smallest_job) <- (job_time +. test.time, test :: job_tests)
+  in
+  (* Finding the optimal partition is NP-complete.
+     We use a heuristic to find an approximation: allocate longest tests first,
+     then fill the gaps with smaller tests. *)
+  let longest_first {time = a; _} {time = b; _} = Float.compare b a in
+  List.iter allocate (List.sort longest_first tests) ;
+  (* Jobs are allocated, now display them. *)
+  let display_job ~negate (total_job_time, job_tests) =
+    print_endline
+      ( String.concat
+          " "
+          (List.map
+             (fun test ->
+               Printf.sprintf
+                 "%s %s"
+                 (if negate then "--not-test" else "--test")
+                 (Log.quote_shell test.title))
+             job_tests)
+      ^ " # "
+      ^ string_of_float total_job_time
+      ^ "s" )
+  in
+  let all_other_tests = ref [] in
+  for i = 0 to job_count - 2 do
+    display_job ~negate:false jobs.(i) ;
+    List.iter
+      (fun test -> all_other_tests := test :: !all_other_tests)
+      (snd jobs.(i))
+  done ;
+  (* The last job uses --not-test so that if a test is added and the job list is not
+     updated, the new test is automatically added to the last job.
+     Note: if [job_count] is 1, this actually outputs no --not-test at all
+     since [all_other_tests] is empty, which is consistent
+     because it means to run all tests. *)
+  display_job ~negate:true (fst jobs.(job_count - 1), !all_other_tests)
+
 let register ~__FILE__ ~title ~tags body =
   let file = Filename.basename __FILE__ in
   check_tags tags ;
@@ -414,12 +494,22 @@ let run () =
       prerr_endline
         "You can use --list to get the list of tests and their tags." ) ;
   (* Actually run the tests (or list them). *)
-  match Cli.options.list with
-  | Some format ->
+  match (Cli.options.list, Cli.options.suggest_jobs) with
+  | (Some format, None) ->
       list_tests format
-  | None ->
+  | (None, Some record_file) -> (
+    match read_recorded_results record_file with
+    | exception Sys_error error ->
+        Printf.eprintf "Failed to read record: %s\n%!" error ;
+        exit 1
+    | record ->
+        suggest_jobs record )
+  | (Some _, Some _) ->
+      prerr_endline
+        "Cannot use both --list and --suggest-jobs at the same time."
+  | (None, None) ->
       let rec run iteration =
-        let run_and_measure_time test =
+        let run_and_measure_time (test : test) =
           let start = Unix.gettimeofday () in
           really_run ~iteration test.title test.body ;
           let time = Unix.gettimeofday () -. start in
@@ -429,5 +519,6 @@ let run () =
         if Cli.options.loop then run (iteration + 1)
       in
       run 1 ;
+      Option.iter record_results Cli.options.record ;
       if !a_test_failed then exit 1 ;
       if Cli.options.time then display_time_summary ()
-- 
GitLab


From 45552e99b3273e871bab470ead55433fc69a235c Mon Sep 17 00:00:00 2001
From: Romain Bardou <romain@nomadic-labs.com>
Date: Tue, 30 Mar 2021 18:22:59 +0200
Subject: [PATCH 4/9] Tezt: forbid multiple tests with the same title

---
 tezt/lib/test.ml    | 76 ++++++++++++++++++++++++++++-----------------
 tezt/tests/proxy.ml |  3 +-
 2 files changed, 49 insertions(+), 30 deletions(-)

diff --git a/tezt/lib/test.ml b/tezt/lib/test.ml
index 5424d328d174..312b679654e1 100644
--- a/tezt/lib/test.ml
+++ b/tezt/lib/test.ml
@@ -263,25 +263,32 @@ type test = {
   mutable time : float;
 }
 
-(* List of tests added using [add] and that match command-line filters. *)
-let list : test list ref = ref []
+(* Tests added using [register] and that match command-line filters. *)
+let registered : test String_map.t ref = ref String_map.empty
+
+(* Using [iter_registered] instead of [String_map.iter] allows to more easily
+   change the representation of [registered] in the future if needed. *)
+let iter_registered f = String_map.iter (fun _ -> f) !registered
+
+let fold_registered acc f =
+  String_map.fold (fun _ test acc -> f acc test) !registered acc
+
+(* Map [register] as if it was a list, to obtain a list. *)
+let map_registered_list f = fold_registered [] @@ fun acc test -> f test :: acc
 
 let list_tests format =
   match format with
   | `Tsv ->
-      List.iter
-        (fun {file; title; tags; _} ->
-          Printf.printf "%s\t%s\t%s\n%!" file title (String.concat " " tags))
-        !list
+      iter_registered
+      @@ fun {file; title; tags; _} ->
+      Printf.printf "%s\t%s\t%s\n%!" file title (String.concat " " tags)
   | `Ascii_art ->
       let file_header = "FILE" in
       let title_header = "TITLE" in
       let tags_header = "TAGS" in
       let list =
-        List.map
-          (fun {file; title; tags; _} ->
-            (file, title, String.concat ", " tags))
-          !list
+        map_registered_list
+        @@ fun {file; title; tags; _} -> (file, title, String.concat ", " tags)
       in
       (* Compute the size of each column. *)
       let (file_size, title_size, tags_size) =
@@ -336,18 +343,14 @@ let list_tests format =
       ()
 
 let display_time_summary () =
-  let sum_time = List.fold_left (fun acc {time; _} -> acc +. time) 0. in
-  let total_time = sum_time !list in
+  let total_time = fold_registered 0. @@ fun acc {time; _} -> acc +. time in
   let tests_by_file =
-    List.fold_left
-      (fun acc test ->
-        String_map.add
-          test.file
-          ( test
-          :: (String_map.find_opt test.file acc |> Option.value ~default:[]) )
-          acc)
-      String_map.empty
-      !list
+    fold_registered String_map.empty
+    @@ fun acc test ->
+    String_map.add
+      test.file
+      (test :: (String_map.find_opt test.file acc |> Option.value ~default:[]))
+      acc
   in
   let show_time seconds =
     let seconds = int_of_float seconds in
@@ -363,7 +366,10 @@ let display_time_summary () =
       title
   in
   let print_time_for_file file tests =
-    print_time "" file (sum_time tests) ;
+    print_time
+      ""
+      file
+      (List.fold_left (fun acc {time; _} -> acc +. time) 0. tests) ;
     List.iter (fun {title; time; _} -> print_time "- " title time) tests
   in
   String_map.iter print_time_for_file tests_by_file ;
@@ -379,9 +385,8 @@ type marshaled_test = {
 let record_results filename =
   (* Remove the closure ([body]). *)
   let marshaled_tests =
-    List.map
-      (fun {file; title; tags; body = _; time} -> {file; title; tags; time})
-      !list
+    map_registered_list
+    @@ fun {file; title; tags; body = _; time} -> {file; title; tags; time}
   in
   (* Write to file using Marshal.
      This is not very robust but enough for the purposes of this file. *)
@@ -451,12 +456,27 @@ let suggest_jobs (tests : marshaled_test list) =
 
 let register ~__FILE__ ~title ~tags body =
   let file = Filename.basename __FILE__ in
+  ( match String_map.find_opt title !registered with
+  | None ->
+      ()
+  | Some {file = other_file; tags = other_tags; _} ->
+      Printf.eprintf "Error: there are several tests with title: %S\n" title ;
+      Printf.eprintf
+        "- first seen in: %s with tags: %s\n"
+        other_file
+        (String.concat ", " other_tags) ;
+      Printf.eprintf
+        "- also seen in: %s with tags: %s\n%!"
+        file
+        (String.concat ", " tags) ;
+      exit 1 ) ;
   check_tags tags ;
   register_file file ;
   register_title title ;
   List.iter register_tag tags ;
   if test_should_be_run ~file ~title ~tags then
-    list := {file; title; tags; body; time = 0.} :: !list
+    let test = {file; title; tags; body; time = 0.} in
+    registered := String_map.add title test !registered
 
 let run () =
   (* Now that all tests are registered, put them in registration order. *)
@@ -477,7 +497,7 @@ let run () =
   if (not all_files_exist) || (not all_titles_exist) || not all_tags_exist then
     exit 1 ;
   (* Print a warning if no test was selected. *)
-  if !list = [] then (
+  if String_map.is_empty !registered then (
     Printf.eprintf
       "No test found for filters: %s\n%!"
       (String.concat
@@ -515,7 +535,7 @@ let run () =
           let time = Unix.gettimeofday () -. start in
           test.time <- test.time +. time
         in
-        List.iter run_and_measure_time !list ;
+        iter_registered run_and_measure_time ;
         if Cli.options.loop then run (iteration + 1)
       in
       run 1 ;
diff --git a/tezt/tests/proxy.ml b/tezt/tests/proxy.ml
index 7906d51f495e..e2f7d01a4e28 100644
--- a/tezt/tests/proxy.ml
+++ b/tezt/tests/proxy.ml
@@ -104,8 +104,7 @@ let test_cache_at_most_once ?query_string path =
 
 let test_cache_at_most_once ~protocols =
   let paths =
-    [ (["context"; "constants"], []);
-      (["helpers"; "baking_rights"], []);
+    [ (["helpers"; "baking_rights"], []);
       (["helpers"; "baking_rights"], [("all", "true")]);
       (["helpers"; "current_level"], []);
       (["minimal_valid_time"], []);
-- 
GitLab


From a31ea7cfe8761bc5ecaa53594b8163ad8daa226f Mon Sep 17 00:00:00 2001
From: Romain Bardou <romain@nomadic-labs.com>
Date: Wed, 31 Mar 2021 14:47:03 +0200
Subject: [PATCH 5/9] Tezt: if a test/file/tag does not exist, warn instead of
 exiting

This is important if we start using --suggest-jobs: if a test is removed,
we don't necessarily want to have to update the job split (which requires running all tests
to measure time).
---
 tezt/lib/test.ml | 31 +++++++++----------------------
 1 file changed, 9 insertions(+), 22 deletions(-)

diff --git a/tezt/lib/test.ml b/tezt/lib/test.ml
index 312b679654e1..92658c4cba34 100644
--- a/tezt/lib/test.ml
+++ b/tezt/lib/test.ml
@@ -245,14 +245,9 @@ let register_tag tag = known_tags := String_set.add tag !known_tags
 
 (* Check that all [specified] values are in [!known]. *)
 let check_existence kind known specified =
-  match
-    String_set.elements (String_set.diff (String_set.of_list specified) !known)
-  with
-  | [] ->
-      true
-  | unknown ->
-      List.iter (Printf.eprintf "Unknown %s: %s\n" kind) unknown ;
-      false
+  String_set.iter
+    (Log.warn "Unknown %s: %s" kind)
+    (String_set.diff (String_set.of_list specified) !known)
 
 (* Field [time] contains the cumulated time taken by all successful runs of this test. *)
 type test = {
@@ -482,20 +477,12 @@ let run () =
   (* Now that all tests are registered, put them in registration order. *)
   list := List.rev !list ;
   (* Check command-line options. *)
-  let all_files_exist =
-    check_existence "--file" known_files Cli.options.files_to_run
-  in
-  let all_titles_exist =
-    check_existence "--test" known_titles Cli.options.tests_to_run
-  in
-  let all_tags_exist =
-    check_existence
-      "tag"
-      known_tags
-      (Cli.options.tags_to_run @ Cli.options.tags_not_to_run)
-  in
-  if (not all_files_exist) || (not all_titles_exist) || not all_tags_exist then
-    exit 1 ;
+  check_existence "--file" known_files Cli.options.files_to_run ;
+  check_existence "--test" known_titles Cli.options.tests_to_run ;
+  check_existence
+    "tag"
+    known_tags
+    (Cli.options.tags_to_run @ Cli.options.tags_not_to_run) ;
   (* Print a warning if no test was selected. *)
   if String_map.is_empty !registered then (
     Printf.eprintf
-- 
GitLab


From f7d259b2176234a80173a18d2f4d5b99a8ea22f8 Mon Sep 17 00:00:00 2001
From: Romain Bardou <romain@nomadic-labs.com>
Date: Wed, 31 Mar 2021 15:05:38 +0200
Subject: [PATCH 6/9] Tezt: add --loop-count

---
 tezt/lib/cli.ml  | 20 ++++++++++++++++----
 tezt/lib/cli.mli |  5 ++++-
 tezt/lib/log.ml  |  5 +++--
 tezt/lib/test.ml | 20 ++++++++++++--------
 4 files changed, 35 insertions(+), 15 deletions(-)

diff --git a/tezt/lib/cli.ml b/tezt/lib/cli.ml
index 6f030281e366..725bdabc6660 100644
--- a/tezt/lib/cli.ml
+++ b/tezt/lib/cli.ml
@@ -28,6 +28,8 @@ type log_level = Quiet | Error | Warn | Report | Info | Debug
 
 type temporary_file_mode = Delete | Delete_if_successful | Keep
 
+type loop_mode = Infinite | Count of int
+
 type options = {
   color : bool;
   log_level : log_level;
@@ -45,7 +47,7 @@ type options = {
   global_timeout : float option;
   test_timeout : float option;
   reset_regressions : bool;
-  loop : bool;
+  loop_mode : loop_mode;
   time : bool;
   starting_port : int;
   record : string option;
@@ -72,7 +74,7 @@ let options =
   let global_timeout = ref None in
   let test_timeout = ref None in
   let reset_regressions = ref false in
-  let loop = ref false in
+  let loop_mode = ref (Count 1) in
   let time = ref false in
   let starting_port = ref 16384 in
   let record = ref None in
@@ -98,6 +100,10 @@ let options =
     if value < 1 then raise (Arg.Bad "--job-count must be positive") ;
     job_count := value
   in
+  let set_loop_count value =
+    if value < 0 then raise (Arg.Bad "--loop-count must be positive or null") ;
+    loop_mode := Count value
+  in
   let spec =
     Arg.align
       [ ("--color", Arg.Set color, " Use colors in output.");
@@ -189,12 +195,18 @@ let options =
           " Remove regression test outputs if they exist, and regenerate them."
         );
         ( "--loop",
-          Arg.Set loop,
+          Arg.Unit (fun () -> loop_mode := Infinite),
           " Restart from the beginning once all tests are done. All tests are \
            repeated until one of them fails or if you interrupt with Ctrl+C. \
            This is useful to reproduce non-deterministic failures. When used \
            in conjunction with --keep-going, tests are repeated even if they \
            fail, until you interrupt them with Ctrl+C." );
+        ( "--loop-count",
+          Arg.Int set_loop_count,
+          "<COUNT> Same as --loop, but stop after all tests have been run \
+           COUNT times. A value of 0 means tests are not run. The default \
+           behavior corresponds to --loop-count 1. If you specify both --loop \
+           and --loop-count, only the last one is taken into account." );
         ( "--time",
           Arg.Set time,
           " Print a summary of the time taken by each test. Ignored if a test \
@@ -272,7 +284,7 @@ let options =
     global_timeout = !global_timeout;
     test_timeout = !test_timeout;
     reset_regressions = !reset_regressions;
-    loop = !loop;
+    loop_mode = !loop_mode;
     time = !time;
     starting_port = !starting_port;
     record = !record;
diff --git a/tezt/lib/cli.mli b/tezt/lib/cli.mli
index 2d7781694a6b..623456bd1df1 100644
--- a/tezt/lib/cli.mli
+++ b/tezt/lib/cli.mli
@@ -58,6 +58,9 @@ type log_level = Quiet | Error | Warn | Report | Info | Debug
 (** What to do with temporary files after the test is finished. *)
 type temporary_file_mode = Delete | Delete_if_successful | Keep
 
+(** How many times to loop. *)
+type loop_mode = Infinite | Count of int
+
 (** Command-line options. *)
 type options = {
   color : bool;
@@ -76,7 +79,7 @@ type options = {
   global_timeout : float option;
   test_timeout : float option;
   reset_regressions : bool;
-  loop : bool;
+  loop_mode : loop_mode;
   time : bool;
   starting_port : int;
   record : string option;
diff --git a/tezt/lib/log.ml b/tezt/lib/log.ml
index 32900f0bdac6..e10da98e90df 100644
--- a/tezt/lib/log.ml
+++ b/tezt/lib/log.ml
@@ -49,7 +49,7 @@ let channel =
         global_timeout = _;
         test_timeout = _;
         reset_regressions = _;
-        loop = _;
+        loop_mode = _;
         time = _;
         starting_port = _;
         record = _;
@@ -273,7 +273,8 @@ let test_result ~iteration test_result test_name =
         ("ABORTED", Color.(FG.red ++ bold))
   in
   let message =
-    if Cli.options.loop then Printf.sprintf "(loop %d) %s" iteration test_name
+    if Cli.options.loop_mode <> Count 1 then
+      Printf.sprintf "(loop %d) %s" iteration test_name
     else test_name
   in
   log_string ~level:Report ~prefix ~prefix_color message
diff --git a/tezt/lib/test.ml b/tezt/lib/test.ml
index 92658c4cba34..8e7e794fc700 100644
--- a/tezt/lib/test.ml
+++ b/tezt/lib/test.ml
@@ -516,14 +516,18 @@ let run () =
         "Cannot use both --list and --suggest-jobs at the same time."
   | (None, None) ->
       let rec run iteration =
-        let run_and_measure_time (test : test) =
-          let start = Unix.gettimeofday () in
-          really_run ~iteration test.title test.body ;
-          let time = Unix.gettimeofday () -. start in
-          test.time <- test.time +. time
-        in
-        iter_registered run_and_measure_time ;
-        if Cli.options.loop then run (iteration + 1)
+        match Cli.options.loop_mode with
+        | Count n when n < iteration ->
+            ()
+        | _ ->
+            let run_and_measure_time (test : test) =
+              let start = Unix.gettimeofday () in
+              really_run ~iteration test.title test.body ;
+              let time = Unix.gettimeofday () -. start in
+              test.time <- test.time +. time
+            in
+            iter_registered run_and_measure_time ;
+            run (iteration + 1)
       in
       run 1 ;
       Option.iter record_results Cli.options.record ;
-- 
GitLab


From 546b78967ddaad73e5d06421bc82acdb73e17675 Mon Sep 17 00:00:00 2001
From: Romain Bardou <romain@nomadic-labs.com>
Date: Wed, 31 Mar 2021 15:10:45 +0200
Subject: [PATCH 7/9] Tezt: average times when using --loop-count

---
 tezt/lib/cli.ml  | 3 ++-
 tezt/lib/test.ml | 7 +++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/tezt/lib/cli.ml b/tezt/lib/cli.ml
index 725bdabc6660..b7ae35868f3d 100644
--- a/tezt/lib/cli.ml
+++ b/tezt/lib/cli.ml
@@ -217,7 +217,8 @@ let options =
         ( "--record",
           Arg.String (fun file -> record := Some file),
           "<FILE> Record test results to FILE. This file can then be used \
-           with --suggest-jobs." );
+           with --suggest-jobs. If you use --loop or --loop-count, times are \
+           averaged for each test." );
         ( "--job-count",
           Arg.Int set_job_count,
           "<COUNT> Set the number of target jobs for --suggest-jobs (default \
diff --git a/tezt/lib/test.ml b/tezt/lib/test.ml
index 8e7e794fc700..eca20bcd92e2 100644
--- a/tezt/lib/test.ml
+++ b/tezt/lib/test.ml
@@ -256,6 +256,7 @@ type test = {
   tags : string list;
   body : unit -> unit Lwt.t;
   mutable time : float;
+  mutable run_count : int;
 }
 
 (* Tests added using [register] and that match command-line filters. *)
@@ -381,7 +382,8 @@ let record_results filename =
   (* Remove the closure ([body]). *)
   let marshaled_tests =
     map_registered_list
-    @@ fun {file; title; tags; body = _; time} -> {file; title; tags; time}
+    @@ fun {file; title; tags; body = _; time; run_count} ->
+    {file; title; tags; time = time /. float (max 1 run_count)}
   in
   (* Write to file using Marshal.
      This is not very robust but enough for the purposes of this file. *)
@@ -470,7 +472,7 @@ let register ~__FILE__ ~title ~tags body =
   register_title title ;
   List.iter register_tag tags ;
   if test_should_be_run ~file ~title ~tags then
-    let test = {file; title; tags; body; time = 0.} in
+    let test = {file; title; tags; body; time = 0.; run_count = 0} in
     registered := String_map.add title test !registered
 
 let run () =
@@ -524,6 +526,7 @@ let run () =
               let start = Unix.gettimeofday () in
               really_run ~iteration test.title test.body ;
               let time = Unix.gettimeofday () -. start in
+              test.run_count <- test.run_count + 1 ;
               test.time <- test.time +. time
             in
             iter_registered run_and_measure_time ;
-- 
GitLab


From 45660fd31c19c78671a0d0599b4e246667064b47 Mon Sep 17 00:00:00 2001
From: Romain Bardou <romain@nomadic-labs.com>
Date: Thu, 22 Apr 2021 15:45:22 +0200
Subject: [PATCH 8/9] Tezt: keep order of registration

---
 tezt/lib/test.ml | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/tezt/lib/test.ml b/tezt/lib/test.ml
index eca20bcd92e2..e8b5af92fabc 100644
--- a/tezt/lib/test.ml
+++ b/tezt/lib/test.ml
@@ -249,8 +249,10 @@ let check_existence kind known specified =
     (Log.warn "Unknown %s: %s" kind)
     (String_set.diff (String_set.of_list specified) !known)
 
-(* Field [time] contains the cumulated time taken by all successful runs of this test. *)
+(* Field [id] is used to be able to iterate on tests in order of registration.
+   Field [time] contains the cumulated time taken by all successful runs of this test. *)
 type test = {
+  id : int;
   file : string;
   title : string;
   tags : string list;
@@ -264,13 +266,23 @@ let registered : test String_map.t ref = ref String_map.empty
 
 (* Using [iter_registered] instead of [String_map.iter] allows to more easily
    change the representation of [registered] in the future if needed. *)
-let iter_registered f = String_map.iter (fun _ -> f) !registered
+let iter_registered f =
+  let list = ref [] in
+  String_map.iter (fun _ test -> list := test :: !list) !registered ;
+  let by_id {id = a; _} {id = b; _} = Int.compare a b in
+  list := List.sort by_id !list ;
+  List.iter (fun test -> f test) !list
 
 let fold_registered acc f =
   String_map.fold (fun _ test acc -> f acc test) !registered acc
 
 (* Map [register] as if it was a list, to obtain a list. *)
-let map_registered_list f = fold_registered [] @@ fun acc test -> f test :: acc
+let map_registered_list f =
+  let list = ref [] in
+  (* By using [iter_registered] we ensure the resulting list is
+     in order of registration. *)
+  (iter_registered @@ fun test -> list := f test :: !list) ;
+  List.rev !list
 
 let list_tests format =
   match format with
@@ -382,7 +394,7 @@ let record_results filename =
   (* Remove the closure ([body]). *)
   let marshaled_tests =
     map_registered_list
-    @@ fun {file; title; tags; body = _; time; run_count} ->
+    @@ fun {id = _; file; title; tags; body = _; time; run_count} ->
     {file; title; tags; time = time /. float (max 1 run_count)}
   in
   (* Write to file using Marshal.
@@ -451,6 +463,8 @@ let suggest_jobs (tests : marshaled_test list) =
      because it means to run all tests. *)
   display_job ~negate:true (fst jobs.(job_count - 1), !all_other_tests)
 
+let next_id = ref 0
+
 let register ~__FILE__ ~title ~tags body =
   let file = Filename.basename __FILE__ in
   ( match String_map.find_opt title !registered with
@@ -471,13 +485,13 @@ let register ~__FILE__ ~title ~tags body =
   register_file file ;
   register_title title ;
   List.iter register_tag tags ;
+  let id = !next_id in
+  incr next_id ;
   if test_should_be_run ~file ~title ~tags then
-    let test = {file; title; tags; body; time = 0.; run_count = 0} in
+    let test = {id; file; title; tags; body; time = 0.; run_count = 0} in
     registered := String_map.add title test !registered
 
 let run () =
-  (* Now that all tests are registered, put them in registration order. *)
-  list := List.rev !list ;
   (* Check command-line options. *)
   check_existence "--file" known_files Cli.options.files_to_run ;
   check_existence "--test" known_titles Cli.options.tests_to_run ;
-- 
GitLab


From 6aeae89007886b5e29880f7d63b3d9cba949e412 Mon Sep 17 00:00:00 2001
From: Romain Bardou <romain@nomadic-labs.com>
Date: Wed, 31 Mar 2021 11:57:03 +0200
Subject: [PATCH 9/9] CI: use --suggest-jobs to generate .gitlab/ci/tezt.yml

---
 .gitlab/ci/tezt.yml          | 34 ++++++--------
 scripts/run-tezt-tests-ci.sh | 60 ------------------------
 scripts/update_tezt_test.ml  | 91 ++++++++++++++++++++++++++++++++++++
 3 files changed, 106 insertions(+), 79 deletions(-)
 delete mode 100755 scripts/run-tezt-tests-ci.sh
 create mode 100755 scripts/update_tezt_test.ml

diff --git a/.gitlab/ci/tezt.yml b/.gitlab/ci/tezt.yml
index 008a33476e84..2fc78f4e9cba 100644
--- a/.gitlab/ci/tezt.yml
+++ b/.gitlab/ci/tezt.yml
@@ -1,34 +1,30 @@
 # We use `scripts/run-tezt-tests-ci.sh` to split the Tezt tests into ranges:
 
-tezt:run-1-33:
+.tezt_template:
   extends: .integration_template
-  script:
-    - sh scripts/run-tezt-tests-ci.sh 1 33
   artifacts:
     paths:
       - tezt.log
     expire_in: 1 day
     when: on_failure
 
-tezt:run-34-66:
-  extends: .integration_template
+# this section is updated using the script scripts/update_tezt_test.sh
+##BEGIN_TEZTTEST##
+tezt:1:
+  extends: .tezt_template
   script:
-    - sh scripts/run-tezt-tests-ci.sh 34 66
-  artifacts:
-    paths:
-      - tezt.log
-    expire_in: 1 day
-    when: on_failure
+    - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --time --test ''Edo: current encoding regression test: level'' --test ''Alpha: alpha encoding regression test: level'' --test ''Alpha: alpha encoding regression test: operation.raw'' --test ''Alpha: alpha encoding regression test: seed'' --test ''Edo: current encoding regression test: seed'' --test ''Alpha: alpha encoding regression test: timestamp'' --test ''Edo: current encoding regression test: fitness'' --test ''Edo: current encoding regression test: block_header'' --test ''Edo: current encoding regression test: block_header.raw'' --test ''Alpha: alpha encoding regression test: nonce'' --test ''Edo: current encoding regression test: period'' --test ''Edo: current encoding regression test: cycle'' --test ''Edo: current encoding regression test: gas.cost'' --test ''Edo: current encoding regression test: voting_period'' --test ''Alpha: alpha encoding regression test: period'' --test ''Edo: current encoding regression test: vote.ballot'' --test ''Alpha: alpha encoding regression test: gas'' --test ''Alpha: alpha encoding regression test: voting_period.kind'' --test ''Alpha: (Mockup) Transfer same participants (asynchronous)'' --test ''Edo: (Mockup) Transfer same participants (asynchronous)'' --test ''Alpha: normalize data (mockup)'' --test ''Edo: current encoding regression test: operation.internal'' --test ''Alpha: (Mockup) origination fees from unrevealed'' --test ''Alpha: alpha encoding regression test: receipt.balance_updates'' --test ''Alpha: hash data ... of type ... (bad)'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --test ''(Mockup) Migration (transfer)'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_proposal) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --test ''Alpha: normalize data'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) Cache at most once'' --test ''Edo: (Proxy) Wrong proto'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --test ''Edo: current encoding regression test: operation.unsigned'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) Cache at most once'' --test ''Alpha: alpha (mode client) RPC regression tests: others'' --test ''Alpha: alpha encoding regression test: operation'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --test ''Alpha: node initialization (full mode)'' --test ''Edo: (Proxy) Bake'' --test ''Alpha: alpha (mode client) RPC regression tests: delegates'' --test ''Alpha: alpha (mode proxy) RPC regression tests: delegates'' --test ''Edo: current (mode client) RPC regression tests: votes'' --test ''Alpha: baking ordering'' --test ''Edo: (Proxy) Compare RPC get'' --test ''tezos-codec dump encodings'' --test ''Alpha: alpha (mode proxy) RPC regression tests: contracts'' --test ''Alpha: (Mockup) Multi transfer/multi baking (asynchronous)'' --test ''Alpha: node synchronization (rolling / full)'' --test ''Alpha: node synchronization (archive / archive)'' --test ''Alpha: node synchronization (full / full)'' # 304.566508055s'
 
-tezt:run-67-end:
-  extends: .integration_template
+tezt:2:
+  extends: .tezt_template
   script:
-    - sh scripts/run-tezt-tests-ci.sh 67
-  artifacts:
-    paths:
-      - tezt.log
-    expire_in: 1 day
-    when: on_failure
+    - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --time --test ''Alpha: alpha encoding regression test: delegate.frozen_balance'' --test ''Edo: current encoding regression test: nonce'' --test ''Alpha: alpha encoding regression test: fitness'' --test ''Alpha: alpha encoding regression test: vote.ballots'' --test ''Alpha: alpha encoding regression test: block_header'' --test ''Edo: current encoding regression test: vote.ballots'' --test ''Alpha: alpha encoding regression test: block_header.raw'' --test ''Edo: current encoding regression test: timestamp'' --test ''Edo: current encoding regression test: contract'' --test ''Alpha: alpha encoding regression test: cycle'' --test ''Alpha: alpha encoding regression test: raw_level'' --test ''Alpha: alpha encoding regression test: tez'' --test ''Alpha: alpha encoding regression test: gas.cost'' --test ''Alpha: alpha encoding regression test: contract'' --test ''Alpha: (Mockup) RPC header/shell'' --test ''Alpha: alpha encoding regression test: vote.ballot'' --test ''Edo: current encoding regression test: voting_period.kind'' --test ''Alpha: alpha encoding regression test: contract.big_map_diff'' --test ''Edo: current encoding regression test: delegate.balance_updates'' --test ''Alpha: (Mockup) Transfer (asynchronous)'' --test ''Alpha: alpha encoding regression test: operation.internal'' --test ''CLI under connections cap'' --test ''Edo: (Mockup) Transfer'' --test ''Alpha: (Mockup) Transfer'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_period_kind) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_proposal) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/context/constants/errors) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_quorum) Cache at most once'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/levels_in_current_cycle) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/context/constants) Cache at most once'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) Cache at most once'' --test ''Edo: current encoding regression test: operation'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) Cache at most once'' --test ''Alpha: alpha (mode proxy) RPC regression tests: others'' --test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --test ''Edo: current (mode proxy) RPC regression tests: others'' --test ''Alpha: (Proxy) RPC get''\''''s location'' --test ''Alpha: node initialization (archive mode)'' --test ''Alpha: check --connection=1 option'' --test ''Edo: (Proxy) Transfer'' --test ''Edo: current (mode proxy) RPC regression tests: delegates'' --test ''Alpha: Check prevalidator start'' --test ''Alpha: alpha (mode proxy) RPC regression tests: votes'' --test p2p-maintenance-init-expected_connections --test ''Alpha: (Proxy) Compare RPC get'' --test ''Alpha: alpha (mode client) RPC regression tests: contracts'' --test ''Alpha: double baking with accuser'' --test ''Edo: (Mockup) Multi transfer/multi baking (asynchronous)'' --test ''Alpha: node synchronization (rolling / archive)'' --test ''Alpha: node synchronization (full / archive)'' --test ''Alpha: node synchronization (archive / rolling)'' # 304.513179541s'
+
+tezt:3:
+  extends: .tezt_template
+  script:
+    - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --time --not-test ''Alpha: node synchronization (archive / rolling)'' --not-test ''Alpha: node synchronization (full / archive)'' --not-test ''Alpha: node synchronization (rolling / archive)'' --not-test ''Edo: (Mockup) Multi transfer/multi baking (asynchronous)'' --not-test ''Alpha: double baking with accuser'' --not-test ''Alpha: alpha (mode client) RPC regression tests: contracts'' --not-test ''Alpha: (Proxy) Compare RPC get'' --not-test p2p-maintenance-init-expected_connections --not-test ''Alpha: alpha (mode proxy) RPC regression tests: votes'' --not-test ''Alpha: Check prevalidator start'' --not-test ''Edo: current (mode proxy) RPC regression tests: delegates'' --not-test ''Edo: (Proxy) Transfer'' --not-test ''Alpha: check --connection=1 option'' --not-test ''Alpha: node initialization (archive mode)'' --not-test ''Alpha: (Proxy) RPC get''\''''s location'' --not-test ''Edo: current (mode proxy) RPC regression tests: others'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --not-test ''Alpha: alpha (mode proxy) RPC regression tests: others'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --not-test ''Edo: current encoding regression test: operation'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/total_voting_power) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/constants) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/levels_in_current_cycle) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_quorum) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/constants/errors) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_proposal) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/current_period_kind) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --not-test ''Alpha: (Mockup) Transfer'' --not-test ''Edo: (Mockup) Transfer'' --not-test ''CLI under connections cap'' --not-test ''Alpha: alpha encoding regression test: operation.internal'' --not-test ''Alpha: (Mockup) Transfer (asynchronous)'' --not-test ''Edo: current encoding regression test: delegate.balance_updates'' --not-test ''Alpha: alpha encoding regression test: contract.big_map_diff'' --not-test ''Edo: current encoding regression test: voting_period.kind'' --not-test ''Alpha: alpha encoding regression test: vote.ballot'' --not-test ''Alpha: (Mockup) RPC header/shell'' --not-test ''Alpha: alpha encoding regression test: contract'' --not-test ''Alpha: alpha encoding regression test: gas.cost'' --not-test ''Alpha: alpha encoding regression test: tez'' --not-test ''Alpha: alpha encoding regression test: raw_level'' --not-test ''Alpha: alpha encoding regression test: cycle'' --not-test ''Edo: current encoding regression test: contract'' --not-test ''Edo: current encoding regression test: timestamp'' --not-test ''Alpha: alpha encoding regression test: block_header.raw'' --not-test ''Edo: current encoding regression test: vote.ballots'' --not-test ''Alpha: alpha encoding regression test: block_header'' --not-test ''Alpha: alpha encoding regression test: vote.ballots'' --not-test ''Alpha: alpha encoding regression test: fitness'' --not-test ''Edo: current encoding regression test: nonce'' --not-test ''Alpha: alpha encoding regression test: delegate.frozen_balance'' --not-test ''Alpha: node synchronization (full / full)'' --not-test ''Alpha: node synchronization (archive / archive)'' --not-test ''Alpha: node synchronization (rolling / full)'' --not-test ''Alpha: (Mockup) Multi transfer/multi baking (asynchronous)'' --not-test ''Alpha: alpha (mode proxy) RPC regression tests: contracts'' --not-test ''tezos-codec dump encodings'' --not-test ''Edo: (Proxy) Compare RPC get'' --not-test ''Alpha: baking ordering'' --not-test ''Edo: current (mode client) RPC regression tests: votes'' --not-test ''Alpha: alpha (mode proxy) RPC regression tests: delegates'' --not-test ''Alpha: alpha (mode client) RPC regression tests: delegates'' --not-test ''Edo: (Proxy) Bake'' --not-test ''Alpha: node initialization (full mode)'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) No useless RPC call'' --not-test ''Alpha: alpha encoding regression test: operation'' --not-test ''Alpha: alpha (mode client) RPC regression tests: others'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) No useless RPC call'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/baking_rights?all=true) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --not-test ''Edo: current encoding regression test: operation.unsigned'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/ballot_list) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/helpers/baking_rights) No useless RPC call'' --not-test ''Edo: (Proxy) Wrong proto'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_period) Cache at most once'' --not-test ''Alpha: normalize data'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/minimal_valid_time) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/successor_period) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/votes/listings) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/ballots) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/proposals) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/current_proposal) Cache at most once'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/votes/successor_period) No useless RPC call'' --not-test ''Alpha: (Proxy) (/chains/main/blocks/head/context/delegates) Cache at most once'' --not-test ''(Mockup) Migration (transfer)'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/context/nonces/3) Cache at most once'' --not-test ''Edo: (Proxy) (/chains/main/blocks/head/helpers/current_level) Cache at most once'' --not-test ''Alpha: hash data ... of type ... (bad)'' --not-test ''Alpha: alpha encoding regression test: receipt.balance_updates'' --not-test ''Alpha: (Mockup) origination fees from unrevealed'' --not-test ''Edo: current encoding regression test: operation.internal'' --not-test ''Alpha: normalize data (mockup)'' --not-test ''Edo: (Mockup) Transfer same participants (asynchronous)'' --not-test ''Alpha: (Mockup) Transfer same participants (asynchronous)'' --not-test ''Alpha: alpha encoding regression test: voting_period.kind'' --not-test ''Alpha: alpha encoding regression test: gas'' --not-test ''Edo: current encoding regression test: vote.ballot'' --not-test ''Alpha: alpha encoding regression test: period'' --not-test ''Edo: current encoding regression test: voting_period'' --not-test ''Edo: current encoding regression test: gas.cost'' --not-test ''Edo: current encoding regression test: cycle'' --not-test ''Edo: current encoding regression test: period'' --not-test ''Alpha: alpha encoding regression test: nonce'' --not-test ''Edo: current encoding regression test: block_header.raw'' --not-test ''Edo: current encoding regression test: block_header'' --not-test ''Edo: current encoding regression test: fitness'' --not-test ''Alpha: alpha encoding regression test: timestamp'' --not-test ''Edo: current encoding regression test: seed'' --not-test ''Alpha: alpha encoding regression test: seed'' --not-test ''Alpha: alpha encoding regression test: operation.raw'' --not-test ''Alpha: alpha encoding regression test: level'' --not-test ''Edo: current encoding regression test: level'' # 304.424400091s'
+##END_TEZTTEST##
 
 tezt:manual:migration:
   extends: .test_template
diff --git a/scripts/run-tezt-tests-ci.sh b/scripts/run-tezt-tests-ci.sh
deleted file mode 100755
index 46519fda6e3d..000000000000
--- a/scripts/run-tezt-tests-ci.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#! /bin/sh
-
-set -e
-
-usage () {
-    cat >&2 <<EOF
-usage: [dry_run=true] $0 <from> [<to>]
-
-This script allows the CI to run a subset of the Tezt tests (without
-hard-to-maintain knowledge of all the tests).
-
-It builds a temporary script containing a big shell command and then either
-displays it when 'dry_run=true' or runs it.
-
-Examples:
-
-    dry_run=true scripts/run-tezt-tests-ci.sh 98
-
-displays a command running the tests in the range [98; last].
-
-    scripts/run-tezt-tests-ci.sh 5 9
-
-runs the tests from the range [5; 9].
-EOF
-}
-
-from_line="$1"
-if [ "$from_line" = "" ] ; then
-    usage
-    exit 2
-fi
-to_line="$2"
-if [ "$to_line" = "" ] ; then
-    to_line=$(dune exec tezt/tests/main.exe -- --list-tsv | wc -l)
-fi
-
-to_run=$(mktemp)
-echo "Using $to_run [ $from_line , $to_line ]" >&2
-
-cat > "$to_run" <<EOF
-dune exec tezt/tests/main.exe -- \\
-     --color --log-buffer-size 5000 \\
-     --log-file tezt.log --global-timeout 3300 --time \\
-EOF
-# This adds a line of the form '--test "<title>" \' for each test in the range:
-dune exec tezt/tests/main.exe -- --list-tsv \
-    | sed -n "$from_line,$to_line p" \
-    | cut -f 2 \
-    | sed "s/\(.*\)/--test \"\1\" \\\\/" >> "$to_run"
-# the previous line ends with '\' so we add an empty variable to help the
-# shell succeed in parsing the command:
-echo '$extra_empty_arg' >> "$to_run"
-
-if [ "$dry_run" = "true" ] ; then
-    echo "Selected from $from_line to $to_line:"
-    cat "$to_run"
-else
-    cat "$to_run"
-    sh "$to_run"
-fi
diff --git a/scripts/update_tezt_test.ml b/scripts/update_tezt_test.ml
new file mode 100755
index 000000000000..38006534dbd2
--- /dev/null
+++ b/scripts/update_tezt_test.ml
@@ -0,0 +1,91 @@
+#!ocaml
+
+(* This script updates .gitlab/ci/tezt.yml using the --suggest-jobs option of Tezt.
+   To use it, first run all Tezt tests with the --record option:
+
+       dune exec tezt/tests/main.exe -- --record <FILENAME>
+
+   (replace <FILENAME> with, for instance, "tezt-record").
+   Then, run Tezt with --suggest-jobs and feed the result to this script:
+
+       dune exec tezt/tests/main.exe -- --suggest-jobs <FILENAME> --job-count 3 \
+           | scripts/update_tezt_test.ml
+
+   Don't forget to "git add .gitlab/ci/tezt.yml". *)
+
+let target = ".gitlab/ci/tezt.yml"
+let part = target ^ ".part"
+
+let () =
+  (* Open target file and temporary output file. *)
+  let input = open_in target in
+  Fun.protect ~finally: (fun () -> close_in input) @@ fun () ->
+  let output = open_out part in
+  Fun.protect
+    ~finally: (fun () -> close_out output; if Sys.file_exists part then Sys.remove part)
+  @@ fun () ->
+
+  (* Copy until BEGIN_TEZTTEST. *)
+  let rec copy_begin () =
+    let line =
+      try
+        input_line input
+      with End_of_file ->
+        prerr_endline "Not found: line equal to ##BEGIN_TEZTTEST##";
+        raise Exit
+    in
+    output_string output line;
+    output_char output '\n';
+    if line <> "##BEGIN_TEZTTEST##" then copy_begin ()
+  in
+  copy_begin ();
+
+  (* Ignore until END_TEZTTEST. *)
+  let rec ignore_middle () =
+    let line =
+      try
+        input_line input
+      with End_of_file ->
+        prerr_endline "Not found: line equal to ##END_TEZTTEST##";
+        raise Exit
+    in
+    if line <> "##END_TEZTTEST##" then ignore_middle ()
+  in
+  ignore_middle ();
+
+  (* Generate jobs, reading each job parameters on standard input. *)
+  let rec generate_jobs index =
+    match read_line () with
+      | exception End_of_file ->
+          ()
+      | line ->
+          if index > 1 then output_char output '\n';
+          output_string output ("tezt:" ^ string_of_int index ^ ":");
+          output_string output {|
+  extends: .tezt_template
+  script:
+    - 'dune exec tezt/tests/main.exe -- --color --log-buffer-size 5000 --log-file tezt.log --global-timeout 3300 --time |};
+          let double_single_quotes s =
+            String.split_on_char '\'' s |> String.concat "''"
+          in
+          output_string output (double_single_quotes line);
+          output_string output "'\n";
+          generate_jobs (index + 1)
+  in
+  generate_jobs 1;
+  output_string output "##END_TEZTTEST##\n";
+
+  (* Copy until end of file. *)
+  let rec copy_end () =
+    match input_line input with
+      | exception End_of_file ->
+          ()
+      | line ->
+          output_string output line;
+          output_char output '\n';
+          copy_end ()
+  in
+  copy_end ();
+
+  (* Everything went well, replace target. *)
+  Sys.rename part target
-- 
GitLab