Ketrew: Workflow Examples

Contents

Menu

(**************************************************************************)
(*    Copyright 2014, 2015:                                               *)
(*          Sebastien Mondet <seb@mondet.org>,                            *)
(*          Leonid Rozenberg <leonidr@gmail.com>,                         *)
(*          Arun Ahuja <aahuja11@gmail.com>,                              *)
(*          Jeff Hammerbacher <jeff.hammerbacher@gmail.com>               *)
(*                                                                        *)
(*  Licensed under the Apache License, Version 2.0 (the "License");       *)
(*  you may not use this file except in compliance with the License.      *)
(*  You may obtain a copy of the License at                               *)
(*                                                                        *)
(*      http://www.apache.org/licenses/LICENSE-2.0                        *)
(*                                                                        *)
(*  Unless required by applicable law or agreed to in writing, software   *)
(*  distributed under the License is distributed on an "AS IS" BASIS,     *)
(*  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or       *)
(*  implied.  See the License for the specific language governing         *)
(*  permissions and limitations under the License.                        *)
(**************************************************************************)

Workflow Examples

This “test” provides a few functions that create increasingly complex workflows. It gets compiled to a command line application that submits the workflows to the Ketrew engine.

First, the mandatory license:

Copyright 2014, Sebastien Mondet <seb@mondet.org>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.  See the License for the specific language governing
permissions and limitations under the License.

And some preliminary OCaml settings:


open Nonstd
module String = Sosa.Native_string
let say fmt = ksprintf (fun s -> printf "%s\n%!" s) fmt

Workflows

Simple Example With LSF

This function is a more “parametrized” version of the example in the README.md file (host and queue come from the command line). It actually calls submit directly itself.


let run_command_with_lsf ~host ~queue cmd =
  let open Ketrew.EDSL in
  let host = Host.parse host in
  Ketrew.Client.submit_workflow (
    workflow_node without_product
      ~name:"run_command_with_lsf"
      ~make:(lsf (Program.sh cmd)
               ~queue ~wall_limit:"1:30" ~processors:(`Min_max (1,1)) ~host)
  )

Daemonize With Nohup/Setsid

This function is like run_command_with_lsf but uses daemonize with the “nohup-setsid” method instead of the batch scheduler.


let run_command_with_nohup ~host cmd =
  let open Ketrew.EDSL in
  let host = Host.parse host in
  Ketrew.Client.submit_workflow (
    workflow_node without_product
      ~name:(sprintf "NhSs: %S" cmd)
      ~make:(daemonize ~using:`Nohup_setsid  (Program.sh cmd) ~host)
  )

This kind of “daemonized” job should work on any decent Unix system.

Daemonize With “The Python Hack”

This function is like run_command_with_nohup but uses the “python daemon” method instead.


let run_command_with_python_method ~host cmd =
  let open Ketrew.EDSL in
  let host = Host.parse host in
  Ketrew.Client.submit_workflow (
    workflow_node without_product
      ~name:(sprintf "Pyd: %S" cmd)
      ~make:(daemonize (Program.sh cmd)
               ~using:`Python_daemon ~host
               ~call_script:(fun script -> ["bash"; "--verbose"; script]))
  )

The `Python_daemon way of daemonizing was hacked together because MacOSX does not support the nohup and setsid commands any more.

Get a Container fron “Apache Yarn”

This function is like run_command_with_lsf but uses the Ketrew.Yarn backend. Yarn comes from the Hadoop ecosystem and is used to request resources.


let run_command_with_yarn ~host cmd =
  let open Ketrew.EDSL in
  let host = Host.parse host in
  let make =
    yarn_distributed_shell 
      ~host ~container_memory:(`GB 12)
      ~timeout:(`Seconds 3600)
      ~application_name:"YarnKetrewExample"
      (* do not put spaces up there, it can break Yarn *)
      (Program.sh cmd)
  in
  Ketrew.Client.submit_workflow (
    workflow_node without_product ~name:(sprintf "Yarn: %S" cmd) ~make
  )

A First Dependency Chain

This function runs a workflow with no less than 2 nodes!

  • node2 depends on node1.
  • We call submit_workflow on node2 (the last one, i.e. the root of the dependency arborescence).
  • node1 will run and if it succeeds node2 will run.

let run_2_commands_with_python_method ~host cmd1 cmd2 =
  let open Ketrew.EDSL in
  let host = Host.parse host in
  let node1 =
    workflow_node without_product
      ~name:(sprintf "Pyd: %S" cmd1)
      ~make:(daemonize ~using:`Python_daemon (Program.sh cmd1) ~host)
  in
  let node2 =
    workflow_node without_product
      ~name:(sprintf "Pyd: %S" cmd2)
      ~edges:[depends_on node1]
      ~make:(daemonize ~using:`Python_daemon (Program.sh cmd2) ~host)
  in
  Ketrew.Client.submit_workflow node2

For example:

<test-exec> two-py /tmp "du -sh $HOME" "du -sh $HOME/tmp/"

will run du -sh in $HOME and then in $HOME/tmp in the host "/tmp" (which is localhost with /tmp as playground).

With a Condition

This function creates a 2-nodes workflow that fails because of the condition if the first target is not ensured by the build-process it runs.

  • the function make_node is partial application of EDSL.workflow_node with an additional argument ~cmd.
  • target_with_condition is a target that is supposed to ensure that impossible_file exists (which ls /tmp won't do).
    • impossible_file is a datastructure representing a file on a given host.
    • The condition is specified with the ~done_when argument.
  • target2 is just a target that depends on target_with_condition.

let fail_because_of_condition ~host =
  let open Ketrew.EDSL in
  let host = Host.parse host in
  let make_node ~cmd =
    workflow_node ~make:(daemonize ~using:`Python_daemon Program.(sh cmd) ~host)
  in
  let target_with_condition =
    let impossible_file = single_file ~host "/some-inexistent-file" in 
    make_node impossible_file
      ~name:"Failing-target"
      ~cmd:"ls /tmp"
  in
  let target2 =
    make_node without_product
      ~name:"Won't-run because of failed dependency"
      ~cmd:"ls /tmp"
      ~edges:[depends_on target_with_condition ]
  in
  Ketrew.Client.submit_workflow target2

The Explorer™ will show the failed targets:

* [1]: Won't-run because of failed dependency
ketrew_2014-09-23-21h55m01s460ms-UTC_994326685
Failed: Dependencies died:
ketrew_2014-09-23-21h55m01s460ms-UTC_089809344 died
* [2]: Failing-target
ketrew_2014-09-23-21h55m01s460ms-UTC_089809344
Failed: Process Failure: the target did not ensure (Volume
{([Host /tmp?shell=sh%2C-c]) (Root: /) (Tree: Single path:
"some-inexistent-file")} exists)

Example “Backup” Workflow

This an example of a “backup” workflow.

Take a directory, make a tar.gz, save its MD5 sum, and if gpg is true, call gpg -c and delete the tar.gz.

The passphrase for GPG must be in a file ~/.backup_passphrase as

  export BACKUP_PASSPHRASE="some passsphraaase"
  

let make_targz_on_host ?(gpg=true) ?dest_prefix ~host ~dir () =
  let open Ketrew.EDSL in
  let run_program: Program.t -> Build_process.t =
    fun p -> daemonize ~using:`Python_daemon ~host p in
  let dest_base =
    match dest_prefix with
    | Some s -> s
    | None -> sprintf  "/tmp/backup_from_ketrew_cli"
  in
  let destination_product ext =
    (* `destination` creates “volume-artifacts” for a given file
       extension. *)
    single_file ~host (sprintf "%s.%s" dest_base ext)
  in
  let make_targz : single_file workflow_node = 
    let product = (destination_product "tar.gz") in
    workflow_node product ~name:"make-tar.gz"
      ~make:(run_program
               Program.(
                 shf "cd %s/../" dir
                 &&
                 shf  "tar cfz '%s' '%s'" product#path (Filename.basename dir)))
  in
  let md5_targz =
    let md5_product = destination_product "md5" in  
    workflow_node md5_product
      ~name:"make-md5-of-tar.gz"
      ~edges:[depends_on  make_targz]
      ~make:(run_program
               Program.(shf "md5sum '%s' > '%s'"
                          make_targz#product#path md5_product#path))
  in
  let gpg_targets () =
    let delete_the_targz =
      workflow_node without_product
        ~name:"rm-tar.gz"
        ~make:(run_program (Program.shf "rm -f '%s'" make_targz#product#path))
    in
    let gpg_file = destination_product "tar.gz.gpg" in
    let make_it =
      workflow_node gpg_file
        ~name:"make-gpg-of-tar.gz"
        ~edges:[
          depends_on make_targz;
          depends_on md5_targz;
          on_success_activate delete_the_targz;
        ]
        ~make:(run_program
                 Program.(
                   sh ". ~/.backup_passphrase"
                   && shf "gpg -c --passphrase $BACKUP_PASSPHRASE -o '%s' '%s'"
                     gpg_file#path make_targz#product#path)) in
    make_it
  in
  let common_ancestor =
    let edges =
      match gpg with
      | false -> [ depends_on md5_targz]
      | true ->  [ depends_on (gpg_targets ())]
    in
    workflow_node without_product
      ~name:"Backup workflow common ancestor" ~edges
  in
  (* By running the common-ancestor we pull and activate all the targets to do. *)
  Ketrew.Client.submit_workflow common_ancestor

Build Ketrew On a Vagrant VM

This function builds completely different workflows depedending on the input command:

  • prepare → prepare a vagrant VM, ready to SSH to
  • go → build Ketrew on the on vagrant VM
  • clean → shutdown and delete the VM

let run_ketrew_on_vagrant what_to_do =
  let open Ketrew.EDSL in
  let (//) = Filename.concat in
  let tmp_dir =  Sys.getenv "HOME"  // "tmp/ketrew_vagrant" in
  let vagrant_tmp = tmp_dir // "vagr_vm" in
  let vagrant_host = Host.parse (tmp_dir // "playground") in
  let do_on_vagrant_host p =
    daemonize ~using:`Python_daemon ~host:vagrant_host p in
  let ssh_config =
    single_file ~host:vagrant_host (tmp_dir // "ssh_config") in
  match what_to_do with
  | `Prepare ->
    let init =
      workflow_node ~name:"init-vagrant" 
        (single_file ~host:vagrant_host (vagrant_tmp // "Vagrantfile"))
        ~make:(do_on_vagrant_host Program.(
            exec ["mkdir"; "-p"; vagrant_tmp]
            && exec ["cd"; vagrant_tmp]
            && exec ["vagrant"; "init"; "hashicorp/precise64"]
          ))
    in
    let running =
      workflow_node without_product ~name:"vagrant-up"
        (* TODO: create a product that uses `vagrant status` *)
        ~edges:[depends_on init]
        ~make:(do_on_vagrant_host Program.(
            exec ["cd"; vagrant_tmp]
            && exec ["vagrant"; "up"]))
    in
    let make_ssh_config =
      workflow_node without_product
        (* not a single-file workflow-node, because we want this file
           regenerated every time *)
        ~name:"vagrant-ssh-config" ~edges:[depends_on running]
        ~make:(do_on_vagrant_host Program.(
            exec ["cd"; vagrant_tmp]
            && shf "vagrant ssh-config --host Vagrew > %s" ssh_config#path))
    in
    make_ssh_config
  | `Go ->
    let vagrant_box =
      let open Ketrew in
      Host.ssh ~add_ssh_options:["-F"; ssh_config#path]
        ~playground:"/tmp/KT/" "Vagrew"
    in
    let do_on_vagrant_box p =
      daemonize ~using:`Nohup_setsid ~host:vagrant_box p in
    let get_c_dependencies =
      workflow_node without_product
        ~name:"apt-get-c-deps"
        ~make:(do_on_vagrant_box Program.(
            sh "echo GO"
            && sh "sudo apt-get update"
            && sh "sudo apt-get install -y m4 build-essential libgdbm-dev"
          ))
    in
    let get_opam =
      workflow_node (single_file ~host:vagrant_box "/usr/bin/opam")
        ~name:"apt-get-opam"
        ~edges:[depends_on get_c_dependencies]
        (* `get_opam` does not really depend on `get_c_dependencies` but
           `apt-get` does not work in parallel, so we need to make things
           sequential. *)
        ~make:(do_on_vagrant_box Program.(
            sh "echo GO"
            && sh "sudo apt-get update"
            && sh "sudo apt-get install -y python-software-properties"
            && sh "sudo add-apt-repository -y ppa:avsm/ocaml41+opam11"
            && sh "sudo apt-get update -qq"
            && sh "sudo apt-get install -qq ocaml ocaml-native-compilers camlp4-extra aspcud opam"
          ))
    in
    let init_opam =
      workflow_node
        (single_file ~host:vagrant_box "/home/vagrant/.opam/opam-init/init.sh")
        ~name:"init-opam"
        ~edges:[depends_on get_opam]
        ~make:(do_on_vagrant_box Program.( sh "opam init"))
    in
    let install_ketrew compiler =
      workflow_node
        (single_file ~host:vagrant_box 
           (sprintf "/home/vagrant/.opam/%s/bin/ketrew" compiler))
        ~name:"opam-install-ketrew"
        ~edges:[depends_on init_opam; depends_on get_c_dependencies]

        ~make:(do_on_vagrant_box Program.(
            shf "opam switch %s" compiler
            && sh "opam remote add smondet https://github.com/smondet/dev-opam-repo.git || echo smondet_remote_already_there"
            && sh "opam remove ocamlfind || echo ocamlfind_not_installed"
            && sh "opam pin ketrew https://github.com/smondet/ketrew/ || echo ketrew_already_pinned"
            && sh "opam install -y ketrew"
          ))
    in
    (* We forget the product so that all branches of the `match..with`
       have the same type *)
    install_ketrew "system" |> forget_product
  | `Clean ->
    let kill =
      workflow_node without_product
        ~name:"kill-vagrant"
        ~make:(do_on_vagrant_host Program.(
            exec ["cd"; vagrant_tmp]
            && exec ["vagrant"; "destroy"; "-f"]
          ))
    in
    let rm_temp =
      workflow_node without_product
        ~name:"rm-temp"
        ~edges:[depends_on kill]
        ~make:(do_on_vagrant_host Program.(exec ["rm"; "-fr"; vagrant_tmp]))
    in
    rm_temp

“Main” Of the Module

Command line parsing for this multi-workflow script.

Here it is kept very basic but one may use usual OCaml tools (Arg module, or Cmdliner -- which is already linked-in with Ketrew).


let () =
  let argl = Array.to_list Sys.argv in
  match List.tl_exn argl with
  | "tgz" :: more_args ->
    begin match more_args with
    | host_uri :: dir :: [] ->
      make_targz_on_host ~host:(Ketrew.EDSL.Host.parse host_uri) ~dir ()
    | host :: dir :: dest_prefix :: [] ->
      make_targz_on_host ~dest_prefix ~host:(Ketrew.EDSL.Host.parse host) ~dir ()
    | host :: dir :: dest_prefix :: "with-gpg" :: []  ->
      make_targz_on_host ~gpg:true ~dest_prefix ~host:(Ketrew.EDSL.Host.parse host) ~dir ()
    | other ->
      say "usage: %s tgz <host> <dir> [dest-prefix] [\"with-gpg\"]" Sys.argv.(0);
      failwith "Wrong command line"
    end
  | "lsf" :: more ->
    begin match more with
    | host :: queue :: cmd :: [] ->
      run_command_with_lsf ~host ~queue cmd
    | other ->
      say "usage: %s lsf <host> <queue> <cmd>" Sys.argv.(0);
      failwith "Wrong command line"
    end
  | "nohup-setsid" :: more
  | "nhss" :: more ->
    begin match more with
    | host :: cmd :: [] -> run_command_with_nohup ~host cmd
    | other ->
      say "usage: %s nhss <host> <cmd>" Sys.argv.(0);
      failwith "Wrong command line"
    end
  | "python-daemon" :: more
  | "pyd" :: more ->
    begin match more with
    | host :: cmd :: [] -> run_command_with_python_method ~host cmd
    | other ->
      say "usage: %s pyd <host> <cmd>" Sys.argv.(0);
      failwith "Wrong command line" end
  | "yarn" :: more ->
    begin match more with
    | host :: cmd :: [] -> run_command_with_yarn ~host cmd
    | other ->
      say "usage: %s yarn <host> <cmd>" Sys.argv.(0);
      failwith "Wrong command line"
    end
  | "two-py" :: more ->
    begin match more with
    | host :: cmd1 :: cmd2 :: [] -> 
      run_2_commands_with_python_method ~host cmd1 cmd2
    | other ->
      say "usage: %s two-py <host> <cmd1> <cmd2>" Sys.argv.(0);
      failwith "Wrong command line"
    end
  | "failing-product" :: host :: [] ->
    fail_because_of_condition ~host
  | "CI" :: more ->
    let add_tags = ["ketrew-on-vagrant"; "workflow-examples" ] @ more in
    begin match more with
    | "prepare" :: [] ->
      Ketrew.Client.submit_workflow (run_ketrew_on_vagrant `Prepare)
        ~add_tags
    | "go" :: [] ->
      Ketrew.Client.submit_workflow (run_ketrew_on_vagrant `Go)
        ~add_tags
    | "clean" :: [] ->
      Ketrew.Client.submit_workflow (run_ketrew_on_vagrant `Clean)
        ~add_tags
    | other ->
      say "usage: %s CI {prepare,go,clean}" Sys.argv.(0);
      failwith "Wrong command line"
    end
  | args ->
    say "usage: %s [website|tgz|lsf|nhss|pyd|two-py|failing-product|CI|pythological] ..."
      Sys.argv.(0);
    say "Don't know what to do with %s" (String.concat ~sep:", " args);
    failwith "Wrong command line"