(**************************************************************************) (* Copyright 2014, 2015: *) (* Sebastien Mondet <seb@mondet.org>, *) (* Leonid Rozenberg <leonidr@gmail.com>, *) (* Arun Ahuja <aahuja11@gmail.com>, *) (* Jeff Hammerbacher <jeff.hammerbacher@gmail.com> *) (* *) (* Licensed under the Apache License, Version 2.0 (the "License"); *) (* you may not use this file except in compliance with the License. *) (* You may obtain a copy of the License at *) (* *) (* http://www.apache.org/licenses/LICENSE-2.0 *) (* *) (* Unless required by applicable law or agreed to in writing, software *) (* distributed under the License is distributed on an "AS IS" BASIS, *) (* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or *) (* implied. See the License for the specific language governing *) (* permissions and limitations under the License. *) (**************************************************************************)
This “test” provides a few functions that create increasingly complex workflows. It gets compiled to a command line application that submits the workflows to the Ketrew engine.
First, the mandatory license:
Copyright 2014, Sebastien Mondet <seb@mondet.org>
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied. See the License for the specific language governing
permissions and limitations under the License.
And some preliminary OCaml settings:
open Nonstd module String = Sosa.Native_string let say fmt = ksprintf (fun s -> printf "%s\n%!" s) fmt
This function is a more “parametrized” version of the example in the
README.md
file (host
and queue
come from the command line).
It actually calls submit
directly itself.
let run_command_with_lsf ~host ~queue cmd = let open Ketrew.EDSL in let host = Host.parse host in Ketrew.Client.submit_workflow ( workflow_node without_product ~name:"run_command_with_lsf" ~make:(lsf (Program.sh cmd) ~queue ~wall_limit:"1:30" ~processors:(`Min_max (1,1)) ~host) )
This function is like run_command_with_lsf
but uses
daemonize
with the “nohup-setsid” method instead of the batch scheduler.
let run_command_with_nohup ~host cmd = let open Ketrew.EDSL in let host = Host.parse host in Ketrew.Client.submit_workflow ( workflow_node without_product ~name:(sprintf "NhSs: %S" cmd) ~make:(daemonize ~using:`Nohup_setsid (Program.sh cmd) ~host) )
This kind of “daemonized” job should work on any decent Unix system.
This function is like run_command_with_nohup
but uses
the “python daemon” method instead.
let run_command_with_python_method ~host cmd = let open Ketrew.EDSL in let host = Host.parse host in Ketrew.Client.submit_workflow ( workflow_node without_product ~name:(sprintf "Pyd: %S" cmd) ~make:(daemonize (Program.sh cmd) ~using:`Python_daemon ~host ~call_script:(fun script -> ["bash"; "--verbose"; script])) )
The `Python_daemon
way of daemonizing was hacked together because
MacOSX does not support the nohup
and setsid
commands any more.
This function is like run_command_with_lsf
but uses the
Ketrew.Yarn
backend.
Yarn
comes from the Hadoop ecosystem and is used to request resources.
let run_command_with_yarn ~host cmd = let open Ketrew.EDSL in let host = Host.parse host in let make = yarn_distributed_shell ~host ~container_memory:(`GB 12) ~timeout:(`Seconds 3600) ~application_name:"YarnKetrewExample" (* do not put spaces up there, it can break Yarn *) (Program.sh cmd) in Ketrew.Client.submit_workflow ( workflow_node without_product ~name:(sprintf "Yarn: %S" cmd) ~make )
This function runs a workflow with no less than 2 nodes!
node2
depends on node1
.submit_workflow
on node2
(the last one, i.e. the root of
the dependency arborescence).node1
will run and if it succeeds node2
will run.let run_2_commands_with_python_method ~host cmd1 cmd2 = let open Ketrew.EDSL in let host = Host.parse host in let node1 = workflow_node without_product ~name:(sprintf "Pyd: %S" cmd1) ~make:(daemonize ~using:`Python_daemon (Program.sh cmd1) ~host) in let node2 = workflow_node without_product ~name:(sprintf "Pyd: %S" cmd2) ~edges:[depends_on node1] ~make:(daemonize ~using:`Python_daemon (Program.sh cmd2) ~host) in Ketrew.Client.submit_workflow node2
For example:
<test-exec> two-py /tmp "du -sh $HOME" "du -sh $HOME/tmp/"
will run du -sh
in $HOME
and then in $HOME/tmp
in the host "/tmp"
(which is localhost
with /tmp
as playground).
This function creates a 2-nodes workflow that fails because of the condition if the first target is not ensured by the build-process it runs.
make_node
is partial application of
EDSL.workflow_node
with an additional argument ~cmd
.target_with_condition
is a target that is supposed to ensure that
impossible_file
exists (which ls /tmp
won't do).impossible_file
is a datastructure representing a file on a given host.~done_when
argument.target2
is just a target that depends on target_with_condition
.let fail_because_of_condition ~host = let open Ketrew.EDSL in let host = Host.parse host in let make_node ~cmd = workflow_node ~make:(daemonize ~using:`Python_daemon Program.(sh cmd) ~host) in let target_with_condition = let impossible_file = single_file ~host "/some-inexistent-file" in make_node impossible_file ~name:"Failing-target" ~cmd:"ls /tmp" in let target2 = make_node without_product ~name:"Won't-run because of failed dependency" ~cmd:"ls /tmp" ~edges:[depends_on target_with_condition ] in Ketrew.Client.submit_workflow target2
The Explorer™ will show the failed targets:
* [1]: Won't-run because of failed dependency
ketrew_2014-09-23-21h55m01s460ms-UTC_994326685
Failed: Dependencies died:
ketrew_2014-09-23-21h55m01s460ms-UTC_089809344 died
* [2]: Failing-target
ketrew_2014-09-23-21h55m01s460ms-UTC_089809344
Failed: Process Failure: the target did not ensure (Volume
{([Host /tmp?shell=sh%2C-c]) (Root: /) (Tree: Single path:
"some-inexistent-file")} exists)
This an example of a “backup” workflow.
Take a directory, make a tar.gz, save its MD5 sum, and if gpg
is true
,
call gpg -c
and delete the tar.gz.
The passphrase for GPG must be in a file ~/.backup_passphrase
as
export BACKUP_PASSPHRASE="some passsphraaase"
let make_targz_on_host ?(gpg=true) ?dest_prefix ~host ~dir () = let open Ketrew.EDSL in let run_program: Program.t -> Build_process.t = fun p -> daemonize ~using:`Python_daemon ~host p in let dest_base = match dest_prefix with | Some s -> s | None -> sprintf "/tmp/backup_from_ketrew_cli" in let destination_product ext = (* `destination` creates “volume-artifacts” for a given file extension. *) single_file ~host (sprintf "%s.%s" dest_base ext) in let make_targz : single_file workflow_node = let product = (destination_product "tar.gz") in workflow_node product ~name:"make-tar.gz" ~make:(run_program Program.( shf "cd %s/../" dir && shf "tar cfz '%s' '%s'" product#path (Filename.basename dir))) in let md5_targz = let md5_product = destination_product "md5" in workflow_node md5_product ~name:"make-md5-of-tar.gz" ~edges:[depends_on make_targz] ~make:(run_program Program.(shf "md5sum '%s' > '%s'" make_targz#product#path md5_product#path)) in let gpg_targets () = let delete_the_targz = workflow_node without_product ~name:"rm-tar.gz" ~make:(run_program (Program.shf "rm -f '%s'" make_targz#product#path)) in let gpg_file = destination_product "tar.gz.gpg" in let make_it = workflow_node gpg_file ~name:"make-gpg-of-tar.gz" ~edges:[ depends_on make_targz; depends_on md5_targz; on_success_activate delete_the_targz; ] ~make:(run_program Program.( sh ". ~/.backup_passphrase" && shf "gpg -c --passphrase $BACKUP_PASSPHRASE -o '%s' '%s'" gpg_file#path make_targz#product#path)) in make_it in let common_ancestor = let edges = match gpg with | false -> [ depends_on md5_targz] | true -> [ depends_on (gpg_targets ())] in workflow_node without_product ~name:"Backup workflow common ancestor" ~edges in (* By running the common-ancestor we pull and activate all the targets to do. *) Ketrew.Client.submit_workflow common_ancestor
This function builds completely different workflows depedending on the input command:
prepare
→ prepare a vagrant VM, ready to SSH togo
→ build Ketrew on the on vagrant VMclean
→ shutdown and delete the VMlet run_ketrew_on_vagrant what_to_do = let open Ketrew.EDSL in let (//) = Filename.concat in let tmp_dir = Sys.getenv "HOME" // "tmp/ketrew_vagrant" in let vagrant_tmp = tmp_dir // "vagr_vm" in let vagrant_host = Host.parse (tmp_dir // "playground") in let do_on_vagrant_host p = daemonize ~using:`Python_daemon ~host:vagrant_host p in let ssh_config = single_file ~host:vagrant_host (tmp_dir // "ssh_config") in match what_to_do with | `Prepare -> let init = workflow_node ~name:"init-vagrant" (single_file ~host:vagrant_host (vagrant_tmp // "Vagrantfile")) ~make:(do_on_vagrant_host Program.( exec ["mkdir"; "-p"; vagrant_tmp] && exec ["cd"; vagrant_tmp] && exec ["vagrant"; "init"; "hashicorp/precise64"] )) in let running = workflow_node without_product ~name:"vagrant-up" (* TODO: create a product that uses `vagrant status` *) ~edges:[depends_on init] ~make:(do_on_vagrant_host Program.( exec ["cd"; vagrant_tmp] && exec ["vagrant"; "up"])) in let make_ssh_config = workflow_node without_product (* not a single-file workflow-node, because we want this file regenerated every time *) ~name:"vagrant-ssh-config" ~edges:[depends_on running] ~make:(do_on_vagrant_host Program.( exec ["cd"; vagrant_tmp] && shf "vagrant ssh-config --host Vagrew > %s" ssh_config#path)) in make_ssh_config | `Go -> let vagrant_box = let open Ketrew in Host.ssh ~add_ssh_options:["-F"; ssh_config#path] ~playground:"/tmp/KT/" "Vagrew" in let do_on_vagrant_box p = daemonize ~using:`Nohup_setsid ~host:vagrant_box p in let get_c_dependencies = workflow_node without_product ~name:"apt-get-c-deps" ~make:(do_on_vagrant_box Program.( sh "echo GO" && sh "sudo apt-get update" && sh "sudo apt-get install -y m4 build-essential libgdbm-dev" )) in let get_opam = workflow_node (single_file ~host:vagrant_box "/usr/bin/opam") ~name:"apt-get-opam" ~edges:[depends_on get_c_dependencies] (* `get_opam` does not really depend on `get_c_dependencies` but `apt-get` does not work in parallel, so we need to make things sequential. *) ~make:(do_on_vagrant_box Program.( sh "echo GO" && sh "sudo apt-get update" && sh "sudo apt-get install -y python-software-properties" && sh "sudo add-apt-repository -y ppa:avsm/ocaml41+opam11" && sh "sudo apt-get update -qq" && sh "sudo apt-get install -qq ocaml ocaml-native-compilers camlp4-extra aspcud opam" )) in let init_opam = workflow_node (single_file ~host:vagrant_box "/home/vagrant/.opam/opam-init/init.sh") ~name:"init-opam" ~edges:[depends_on get_opam] ~make:(do_on_vagrant_box Program.( sh "opam init")) in let install_ketrew compiler = workflow_node (single_file ~host:vagrant_box (sprintf "/home/vagrant/.opam/%s/bin/ketrew" compiler)) ~name:"opam-install-ketrew" ~edges:[depends_on init_opam; depends_on get_c_dependencies] ~make:(do_on_vagrant_box Program.( shf "opam switch %s" compiler && sh "opam remote add smondet https://github.com/smondet/dev-opam-repo.git || echo smondet_remote_already_there" && sh "opam remove ocamlfind || echo ocamlfind_not_installed" && sh "opam pin ketrew https://github.com/smondet/ketrew/ || echo ketrew_already_pinned" && sh "opam install -y ketrew" )) in (* We forget the product so that all branches of the `match..with` have the same type *) install_ketrew "system" |> forget_product | `Clean -> let kill = workflow_node without_product ~name:"kill-vagrant" ~make:(do_on_vagrant_host Program.( exec ["cd"; vagrant_tmp] && exec ["vagrant"; "destroy"; "-f"] )) in let rm_temp = workflow_node without_product ~name:"rm-temp" ~edges:[depends_on kill] ~make:(do_on_vagrant_host Program.(exec ["rm"; "-fr"; vagrant_tmp])) in rm_temp
Command line parsing for this multi-workflow script.
Here it is kept very basic but one may use usual OCaml tools
(Arg
module, or Cmdliner
-- which is already linked-in with Ketrew).
let () = let argl = Array.to_list Sys.argv in match List.tl_exn argl with | "tgz" :: more_args -> begin match more_args with | host_uri :: dir :: [] -> make_targz_on_host ~host:(Ketrew.EDSL.Host.parse host_uri) ~dir () | host :: dir :: dest_prefix :: [] -> make_targz_on_host ~dest_prefix ~host:(Ketrew.EDSL.Host.parse host) ~dir () | host :: dir :: dest_prefix :: "with-gpg" :: [] -> make_targz_on_host ~gpg:true ~dest_prefix ~host:(Ketrew.EDSL.Host.parse host) ~dir () | other -> say "usage: %s tgz <host> <dir> [dest-prefix] [\"with-gpg\"]" Sys.argv.(0); failwith "Wrong command line" end | "lsf" :: more -> begin match more with | host :: queue :: cmd :: [] -> run_command_with_lsf ~host ~queue cmd | other -> say "usage: %s lsf <host> <queue> <cmd>" Sys.argv.(0); failwith "Wrong command line" end | "nohup-setsid" :: more | "nhss" :: more -> begin match more with | host :: cmd :: [] -> run_command_with_nohup ~host cmd | other -> say "usage: %s nhss <host> <cmd>" Sys.argv.(0); failwith "Wrong command line" end | "python-daemon" :: more | "pyd" :: more -> begin match more with | host :: cmd :: [] -> run_command_with_python_method ~host cmd | other -> say "usage: %s pyd <host> <cmd>" Sys.argv.(0); failwith "Wrong command line" end | "yarn" :: more -> begin match more with | host :: cmd :: [] -> run_command_with_yarn ~host cmd | other -> say "usage: %s yarn <host> <cmd>" Sys.argv.(0); failwith "Wrong command line" end | "two-py" :: more -> begin match more with | host :: cmd1 :: cmd2 :: [] -> run_2_commands_with_python_method ~host cmd1 cmd2 | other -> say "usage: %s two-py <host> <cmd1> <cmd2>" Sys.argv.(0); failwith "Wrong command line" end | "failing-product" :: host :: [] -> fail_because_of_condition ~host | "CI" :: more -> let add_tags = ["ketrew-on-vagrant"; "workflow-examples" ] @ more in begin match more with | "prepare" :: [] -> Ketrew.Client.submit_workflow (run_ketrew_on_vagrant `Prepare) ~add_tags | "go" :: [] -> Ketrew.Client.submit_workflow (run_ketrew_on_vagrant `Go) ~add_tags | "clean" :: [] -> Ketrew.Client.submit_workflow (run_ketrew_on_vagrant `Clean) ~add_tags | other -> say "usage: %s CI {prepare,go,clean}" Sys.argv.(0); failwith "Wrong command line" end | args -> say "usage: %s [website|tgz|lsf|nhss|pyd|two-py|failing-product|CI|pythological] ..." Sys.argv.(0); say "Don't know what to do with %s" (String.concat ~sep:", " args); failwith "Wrong command line"