struct
let wget_program ?output_filename url =
KEDSL.Program.exec [
"wget";
"-O"; Option.value output_filename ~default:Filename.(basename url);
url
]
let wget_to_folder
~host ~(run_program : Machine.Make_fun.t)
~test_file ~destination url =
let open KEDSL in
let name = "wget-" ^ Filename.basename destination in
let test_target = destination // test_file in
workflow_node (single_file test_target ~host) ~name
~make:(
run_program ~name
~requirements:(Machine.Make_fun.downloading [])
Program.(
exec ["mkdir"; "-p"; destination]
&& shf "wget %s -P %s"
(Filename.quote url)
(Filename.quote destination)))
~edges:[
on_failure_activate (Remove.path_on_host ~host destination);
]
let wget
~host ~(run_program : Machine.Make_fun.t)
url destination =
let open KEDSL in
let name = "wget-" ^ Filename.basename destination in
workflow_node
(single_file destination ~host) ~name
~make:(
run_program ~name
~requirements:(Machine.Make_fun.downloading [])
Program.(
exec ["mkdir"; "-p"; Filename.dirname destination]
&& shf "wget %s -O %s"
(Filename.quote url) (Filename.quote destination)))
~edges:[
on_failure_activate (Remove.path_on_host ~host destination);
]
let wget_gunzip
~host ~(run_program : Machine.Make_fun.t)
~destination url =
let open KEDSL in
let is_gz = Filename.check_suffix url ".gz" in
if is_gz then (
let name = "gunzip-" ^ Filename.basename (destination ^ ".gz") in
let wgot = wget ~host ~run_program url (destination ^ ".gz") in
workflow_node
(single_file destination ~host)
~edges:[
depends_on (wgot);
on_failure_activate (Remove.path_on_host ~host destination);
]
~name
~make:(
run_program ~name
~requirements:(Machine.Make_fun.stream_processor [])
Program.(shf "gunzip -c %s > %s"
(Filename.quote wgot#product#path)
(Filename.quote destination)))
) else (
wget ~host ~run_program url destination
)
let wget_bunzip2
~host ~(run_program : Machine.Make_fun.t)
~destination url =
let open KEDSL in
let is_bz2 = Filename.check_suffix url ".bz2" in
if is_bz2 then (
let name = "bunzip2-" ^ Filename.basename (destination ^ ".bz2") in
let wgot = wget ~host ~run_program url (destination ^ ".bz2") in
workflow_node
(single_file destination ~host)
~edges:[
depends_on (wgot);
on_failure_activate (Remove.path_on_host ~host destination);
]
~name
~make:(
run_program ~name
~requirements:(Machine.Make_fun.stream_processor [])
Program.(shf "bunzip2 -c %s > %s"
(Filename.quote wgot#product#path)
(Filename.quote destination)))
) else (
wget ~host ~run_program url destination
)
let wget_untar
~host ~(run_program : Machine.Make_fun.t)
~destination_folder ~tar_contains url =
let open KEDSL in
let zip_flags =
let is_gz = Filename.check_suffix url ".gz" in
let is_bzip = Filename.check_suffix url ".bz2" in
if is_gz then "z" else if is_bzip then "j" else ""
in
let tar_filename = (destination_folder // "archive.tar") in
let name = "untar-" ^ tar_filename in
let wgot = wget ~host ~run_program url tar_filename in
let file_in_tar = (destination_folder // tar_contains) in
workflow_node
(single_file file_in_tar ~host)
~edges:[
depends_on (wgot);
on_failure_activate (Remove.path_on_host ~host destination_folder);
]
~name
~make:(
run_program ~name
~requirements:(Machine.Make_fun.stream_processor [])
Program.(
exec ["mkdir"; "-p"; destination_folder]
&& shf "tar -x%s -f %s -C %s"
zip_flags
(Filename.quote wgot#product#path)
(Filename.quote destination_folder)))
type tool_file_location = [
| `Scp of string
| `Wget of string
| `Fail of string
]
let get_tool_file
~identifier
~(run_program : Machine.Make_fun.t)
~host ~install_path
loc =
let open KEDSL in
let rm_path = Remove.path_on_host in
let jar_name =
match loc with
| `Fail s -> sprintf "cannot-get-%s.file" identifier
| `Scp s -> Filename.basename s
| `Wget s -> Filename.basename s in
let local_box_path = install_path // jar_name in
workflow_node (single_file local_box_path ~host)
~name:(sprintf "get-%s" jar_name)
~edges:[
on_failure_activate (rm_path ~host local_box_path)
]
~make:(
run_program
~requirements:[
`Internet_access;
`Self_identification [identifier ^ "-instalation"; jar_name];
]
Program.(
shf "mkdir -p %s" install_path
&& begin match loc with
| `Fail msg ->
shf "echo 'Cannot download file for %s: %s'" identifier msg
&& sh "exit 4"
| `Scp s ->
shf "scp %s %s"
(Filename.quote s) (Filename.quote local_box_path)
| `Wget s ->
shf "wget %s -O %s"
(Filename.quote s) (Filename.quote local_box_path)
end))
let gsutil_cp
~(run_program : Machine.Make_fun.t)
~host ~url ~local_path =
let open KEDSL in
workflow_node (single_file ~host local_path)
~name:(sprintf "GSUtil-CP: %s" (Filename.basename local_path))
~edges:[
on_failure_activate (Remove.path_on_host ~host local_path)
]
~make:(
run_program
~requirements:[
`Internet_access;
`Self_identification ["gsutil-cp"; url];
]
Program.(
shf "mkdir -p %s" (Filename.dirname local_path)
&& exec ["gsutil"; "cp"; url; local_path]
)
)
end