diff --git a/tezt/lib_cloud/cli.ml b/tezt/lib_cloud/cli.ml index dc9fd905da7bc94c916820f86b2d9bc3e332e508..cac2031b1d77b12a1674b3c25a0c7852c2892f94 100644 --- a/tezt/lib_cloud/cli.ml +++ b/tezt/lib_cloud/cli.ml @@ -370,3 +370,13 @@ let binaries_path = "Where to find binaries in the docker image by default (default is: \ '/tmp/tezt-runners')" Types.Agent_configuration.default_gcp_binaries_path + +let log_rotation = + Clap.default_int + ~section + ~long:"log-rotation" + ~description: + "Maximum number of log rotations before removing older log files. \ + Default is 300 if a log-file is specified.\n\ + \ Set to 0 to completely disable log-rotation" + 300 diff --git a/tezt/lib_cloud/cli.mli b/tezt/lib_cloud/cli.mli index 178f8b08b251faeca7b37c96f425e4f92d298498..cf80d28babd10b082c441863700a9148e51905f6 100644 --- a/tezt/lib_cloud/cli.mli +++ b/tezt/lib_cloud/cli.mli @@ -138,3 +138,8 @@ val faketime : string option (** Where to find binaries path by default in the docker image. *) val binaries_path : string + +(** How many log rotation until we remove older logs + Defaults to 300 + Use 0 to disable log-rotation *) +val log_rotation : int diff --git a/tezt/lib_cloud/cloud.ml b/tezt/lib_cloud/cloud.ml index 03575d4c8590d508b550561aaa8ae33435e5eeb7..897af955183581219bef46afed9e86c04b1682d6 100644 --- a/tezt/lib_cloud/cloud.ml +++ b/tezt/lib_cloud/cloud.ml @@ -169,6 +169,47 @@ let orchestrator ?(alerts = []) ?(tasks = []) deployement f = Lwt.return_some prometheus else Lwt.return_none in + (* Enable logrotate if --log_rotation was not set to 0... *) + let logrotate = Env.log_rotation <> 0 in + let* logrotate = + (* ... and if log-file was specified *) + match (logrotate, Tezt.Cli.Logs.file) with + (* If no logfile, do not enable logrotate *) + | _, None -> Lwt.return false + (* If logfile and logrotate, configure each agent *) + | true, Some target_file -> + let* () = + (* Logrotate: write a configuration file inside each container *) + Lwt_list.iter_s + (fun agent -> + Logrotate.write_config + ~name:"tezt-cloud" + ~target_file + ~max_rotations:Env.log_rotation + agent) + agents + in + Lwt.return true + (* If no logrotate, return false *) + | false, _ -> Lwt.return_false + in + let tasks = + if logrotate then + let name = "logrotate" in + (* chronos: triggers logrotate every 4 hours. + if each 4 hours, the criteria to trigger a rotation is met, they + will be rotated. + Note that rotation may involve an important IO stress for the machine *) + let tm = "0 0-23/4 * * *" in + let action () = + Lwt_list.iter_s + (fun agent -> Logrotate.run ~name:"tezt-cloud" agent) + agents + in + let task = Chronos.task ~name ~tm ~action in + task :: tasks + else tasks + in let chronos = if List.is_empty tasks then None else diff --git a/tezt/lib_cloud/dockerfiles/dal.Dockerfile b/tezt/lib_cloud/dockerfiles/dal.Dockerfile index a23a27c04b7a11b428e77819530c26a0862992d7..8e99125669e0e8973d539adc2392b1fa3549252a 100644 --- a/tezt/lib_cloud/dockerfiles/dal.Dockerfile +++ b/tezt/lib_cloud/dockerfiles/dal.Dockerfile @@ -21,6 +21,8 @@ RUN apt-get update && apt-get install -y \ iproute2 \ # Can be used to monitor process individually prometheus-process-exporter \ + # rotation for logs + logrotate \ # emacs can be useful for debugging emacs \ # wget can be used to import snapshots diff --git a/tezt/lib_cloud/env.ml b/tezt/lib_cloud/env.ml index 7b5a64f2f990b07388fc64a1954b96164a9ad1ba..0f43ea86b1aef9243509164a5b2e72b0b6e002ec 100644 --- a/tezt/lib_cloud/env.ml +++ b/tezt/lib_cloud/env.ml @@ -92,6 +92,8 @@ let binaries_path = Cli.binaries_path let process_monitoring = Cli.process_monitoring +let log_rotation = Cli.log_rotation + let init () = if tezt_cloud = "" then Test.fail diff --git a/tezt/lib_cloud/env.mli b/tezt/lib_cloud/env.mli index 70d9334dbfd16900e56758a7170650e1f0cee579..a507c48fe41c2cc0d636569a823cb084e9fea250 100644 --- a/tezt/lib_cloud/env.mli +++ b/tezt/lib_cloud/env.mli @@ -130,6 +130,9 @@ val faketime : string option (** Equivalent to [Cli.binaries_path]. *) val binaries_path : string +(** Equivalent to [Cli.log_rotation] *) +val log_rotation : int + (** [init ()] initialises and deploys a Docker registry using Terraform, only when the [mode] is either [`Host] or [`Cloud]. *) val init : unit -> unit Lwt.t diff --git a/tezt/lib_cloud/logrotate.ml b/tezt/lib_cloud/logrotate.ml new file mode 100644 index 0000000000000000000000000000000000000000..895cd37b7ae86c9f74e1ddae71cb693be9471214 --- /dev/null +++ b/tezt/lib_cloud/logrotate.ml @@ -0,0 +1,55 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2025 Nomadic Labs *) +(* *) +(*****************************************************************************) + +let config name = Format.asprintf "/etc/logrotate.d/%s" name + +let write_config ~name ~target_file ~max_rotations agent = + (* Write file locally *) + let source = Temp.file (Format.asprintf "%s.logrotate" (Agent.name agent)) in + (* Choose a reasonable 300 rotations of 200MB max file + Files after 300 rotations are deleted + The rotate is triggered daily or if they are greater than 200MB *) + Base.with_open_out source (fun oc -> + output_string + oc + (Format.asprintf + "%s {@\n%s" + target_file + {| +# Rotates logs every day + daily +# Rotates the log when it reaches 200 megabytes, even if not time for rotation + maxsize 200M +# Skips rotation if the log file is empty + notifempty +# Doesn't produce an error if the log file is missing + missingok +# Compresses rotated logs using gzip by default + compress +# Copies the log file then truncates the original instead of moving it + copytruncate +# Runs rotation commands with root user and group permissions daily + su root root +# Maximum number of rotations before removing the oldests. +|} + ^ Format.asprintf " rotate %d\n}\n" max_rotations)) ; + Log.info "Teztcloud.Logrotate: written configuration for %s" target_file ; + (* Upload the generated config in the destination /etc/logrotate.d/$name *) + let destination = config name in + Log.info + "Teztcloud.Logrotate: uploading %s to %s:%s" + target_file + (Agent.name agent) + destination ; + let* _ = Agent.copy agent ~source ~destination in + Lwt.return_unit + +let run ~name agent = + let name = config name in + (* Run logrotate as root in the docker container *) + Log.info "Teztcloud.Logrotate: executing task %s" name ; + Agent.docker_run_command agent "logrotate" [name] |> Process.check diff --git a/tezt/lib_cloud/logrotate.mli b/tezt/lib_cloud/logrotate.mli new file mode 100644 index 0000000000000000000000000000000000000000..a4b95cff8dc561a931de183d084c50feb2ec1aca --- /dev/null +++ b/tezt/lib_cloud/logrotate.mli @@ -0,0 +1,19 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2025 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(** [write_config name target_file max_rotations agent] will write a logrotate + configuration file for the rotation of [target_file], will a maximum rotation + of [max_rotations] on the [agent]. *) +val write_config : + name:string -> + target_file:string -> + max_rotations:int -> + Agent.t -> + unit Lwt.t + +(** [run name agent] will run the logrotate [name] task on the [agent] *) +val run : name:string -> Agent.t -> unit Lwt.t