struct
let concat ~(run_with : Machine.t) bunch_of_files ~result_path =
let open KEDSL in
let program =
Program.(
exec ["mkdir"; "-p"; Filename.dirname result_path]
&& shf "cat %s > %s"
(List.map bunch_of_files
~f:(fun o -> Filename.quote o#product#path)
|> String.concat ~sep:" ") result_path
) in
let name =
sprintf "concat-all-%s" (Filename.basename result_path) in
workflow_node
(single_file result_path ~host:Machine.(as_host run_with))
~name
~edges:(
on_failure_activate Remove.(file ~run_with result_path)
:: List.map ~f:depends_on bunch_of_files)
~make:(Machine.run_stream_processor run_with ~name program)
let cat_folder ~host
~(run_program : Machine.Make_fun.t)
?(depends_on=[]) ~files_gzipped ~folder ~destination =
let deps = depends_on in
let open KEDSL in
let name = "cat-folder-" ^ Filename.quote folder in
let edges =
on_failure_activate (Remove.path_on_host ~host destination)
:: List.map ~f:depends_on deps in
if files_gzipped then (
workflow_node (single_file destination ~host)
~edges ~name
~make:(
run_program ~name
Program.(
shf "gunzip -c %s/* > %s" (Filename.quote folder)
(Filename.quote destination)))
) else (
workflow_node
(single_file destination ~host)
~edges ~name
~make:(
run_program ~name
Program.(
shf "cat %s/* > %s" (Filename.quote folder) (Filename.quote destination)))
)
end