namespace System
namespace System.Threading
Multiple items
type EntryPointAttribute =
  inherit Attribute
  new : unit -> EntryPointAttribute

Full name: Microsoft.FSharp.Core.EntryPointAttribute

--------------------
new : unit -> EntryPointAttribute
val main : argv:string [] -> int

Full name: index.main
val argv : string []
val wait : ManualResetEventSlim
Multiple items
type ManualResetEventSlim =
  new : unit -> ManualResetEventSlim + 2 overloads
  member Dispose : unit -> unit
  member IsSet : bool with get, set
  member Reset : unit -> unit
  member Set : unit -> unit
  member SpinCount : int with get, set
  member Wait : unit -> unit + 5 overloads
  member WaitHandle : WaitHandle

Full name: System.Threading.ManualResetEventSlim

--------------------
ManualResetEventSlim() : unit
ManualResetEventSlim(initialState: bool) : unit
ManualResetEventSlim(initialState: bool, spinCount: int) : unit
val cts : CancellationTokenSource
Multiple items
type CancellationTokenSource =
  new : unit -> CancellationTokenSource + 2 overloads
  member Cancel : unit -> unit + 1 overload
  member CancelAfter : delay:TimeSpan -> unit + 1 overload
  member Dispose : unit -> unit
  member IsCancellationRequested : bool
  member Token : CancellationToken
  static member CreateLinkedTokenSource : [<ParamArray>] tokens:CancellationToken[] -> CancellationTokenSource + 1 overload

Full name: System.Threading.CancellationTokenSource

--------------------
CancellationTokenSource() : unit
CancellationTokenSource(delay: TimeSpan) : unit
CancellationTokenSource(millisecondsDelay: int) : unit
namespace System.Runtime
val printfn : format:Printf.TextWriterFormat<'T> -> 'T

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.printfn
CancellationTokenSource.Cancel() : unit
CancellationTokenSource.Cancel(throwOnFirstException: bool) : unit
ManualResetEventSlim.Set() : unit
val pretendService : Async<unit>
val async : AsyncBuilder

Full name: Microsoft.FSharp.Core.ExtraTopLevelOperators.async
Multiple items
type Async
static member AsBeginEnd : computation:('Arg -> Async<'T>) -> ('Arg * AsyncCallback * obj -> IAsyncResult) * (IAsyncResult -> 'T) * (IAsyncResult -> unit)
static member AwaitEvent : event:IEvent<'Del,'T> * ?cancelAction:(unit -> unit) -> Async<'T> (requires delegate and 'Del :> Delegate)
static member AwaitIAsyncResult : iar:IAsyncResult * ?millisecondsTimeout:int -> Async<bool>
static member AwaitTask : task:Task -> Async<unit>
static member AwaitTask : task:Task<'T> -> Async<'T>
static member AwaitWaitHandle : waitHandle:WaitHandle * ?millisecondsTimeout:int -> Async<bool>
static member CancelDefaultToken : unit -> unit
static member Catch : computation:Async<'T> -> Async<Choice<'T,exn>>
static member FromBeginEnd : beginAction:(AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromBeginEnd : arg:'Arg1 * beginAction:('Arg1 * AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromBeginEnd : arg1:'Arg1 * arg2:'Arg2 * beginAction:('Arg1 * 'Arg2 * AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromBeginEnd : arg1:'Arg1 * arg2:'Arg2 * arg3:'Arg3 * beginAction:('Arg1 * 'Arg2 * 'Arg3 * AsyncCallback * obj -> IAsyncResult) * endAction:(IAsyncResult -> 'T) * ?cancelAction:(unit -> unit) -> Async<'T>
static member FromContinuations : callback:(('T -> unit) * (exn -> unit) * (OperationCanceledException -> unit) -> unit) -> Async<'T>
static member Ignore : computation:Async<'T> -> Async<unit>
static member OnCancel : interruption:(unit -> unit) -> Async<IDisposable>
static member Parallel : computations:seq<Async<'T>> -> Async<'T []>
static member RunSynchronously : computation:Async<'T> * ?timeout:int * ?cancellationToken:CancellationToken -> 'T
static member Sleep : millisecondsDueTime:int -> Async<unit>
static member Start : computation:Async<unit> * ?cancellationToken:CancellationToken -> unit
static member StartAsTask : computation:Async<'T> * ?taskCreationOptions:TaskCreationOptions * ?cancellationToken:CancellationToken -> Task<'T>
static member StartChild : computation:Async<'T> * ?millisecondsTimeout:int -> Async<Async<'T>>
static member StartChildAsTask : computation:Async<'T> * ?taskCreationOptions:TaskCreationOptions -> Async<Task<'T>>
static member StartImmediate : computation:Async<unit> * ?cancellationToken:CancellationToken -> unit
static member StartWithContinuations : computation:Async<'T> * continuation:('T -> unit) * exceptionContinuation:(exn -> unit) * cancellationContinuation:(OperationCanceledException -> unit) * ?cancellationToken:CancellationToken -> unit
static member SwitchToContext : syncContext:SynchronizationContext -> Async<unit>
static member SwitchToNewThread : unit -> Async<unit>
static member SwitchToThreadPool : unit -> Async<unit>
static member TryCancelled : computation:Async<'T> * compensation:(OperationCanceledException -> unit) -> Async<'T>
static member CancellationToken : Async<CancellationToken>
static member DefaultCancellationToken : CancellationToken

Full name: Microsoft.FSharp.Control.Async

--------------------
type Async<'T>

Full name: Microsoft.FSharp.Control.Async<_>
static member Async.Sleep : millisecondsDueTime:int -> Async<unit>
static member Async.Start : computation:Async<unit> * ?cancellationToken:CancellationToken -> unit
property CancellationTokenSource.Token: CancellationToken
ManualResetEventSlim.Wait() : unit
ManualResetEventSlim.Wait(millisecondsTimeout: int) : bool
ManualResetEventSlim.Wait(timeout: TimeSpan) : bool
ManualResetEventSlim.Wait(cancellationToken: CancellationToken) : unit
ManualResetEventSlim.Wait(millisecondsTimeout: int, cancellationToken: CancellationToken) : bool
ManualResetEventSlim.Wait(timeout: TimeSpan, cancellationToken: CancellationToken) : bool
val signals : obj []
namespace Mono
val ignore : value:'T -> unit

Full name: Microsoft.FSharp.Core.Operators.ignore
type Type =
  inherit MemberInfo
  member Assembly : Assembly
  member AssemblyQualifiedName : string
  member Attributes : TypeAttributes
  member BaseType : Type
  member ContainsGenericParameters : bool
  member DeclaringMethod : MethodBase
  member DeclaringType : Type
  member Equals : o:obj -> bool + 1 overload
  member FindInterfaces : filter:TypeFilter * filterCriteria:obj -> Type[]
  member FindMembers : memberType:MemberTypes * bindingAttr:BindingFlags * filter:MemberFilter * filterCriteria:obj -> MemberInfo[]
  ...

Full name: System.Type
namespace System.Deployment

Resilient F# in the Linux Ecosystem



Deploy, run, and maintain self-supporting F# application code



Dave Curylo

About me

  • Maintain official Docker images for fsharp and swipl
  • Develop F# software for Virtustream within Dell-EMC
  • Languages nerd really enjoying F# and Prolog
Resilient: able to withstand or recover quickly from difficult conditions


Unlikely events happen all the time.

Ecosystem

Component

Purpose

systemd

keeps processes running by tracking the PID

journald

structured logging, for when things go wrong

docker

standardized, immutable infrastructure containers

kubernetes

distributed scheduler

systemd

A modern solution for process management.

  • Like Windows Services, you can
    • specify startup command
    • auto or manual startup
    • set restart thresholds
  • Additionally applies resource constraints (cgroups)
    • CPU
    • Memory
    • Storage and network I/O

Signals, systemd and you!

When systemd should shutdown your application, it will send a SIGTERM.

There are different ways to handle this depending on the runtime.

  • dotnet - handle AssemblyLoadContext.Unloading event.
  • mono - you must use Mono.Posix and wait for signals.
 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 
16: 
17: 
18: 
19: 
20: 
21: 
22: 
open System
open System.Threading

[<EntryPoint>]
let main argv =
    use wait = new ManualResetEventSlim()
    use cts = new CancellationTokenSource ()
    System.Runtime.Loader.AssemblyLoadContext.Default.add_Unloading(
        fun ctx -> 
            printfn "Shutting down nicely!"
            cts.Cancel ()
            wait.Set ()
        )
    let pretendService = async {
        while true do
            printfn "Hello World from F#!"
            do! Async.Sleep 5000
            // In real life we would be starting our service.
    }
    Async.Start (pretendService, cts.Token)
    wait.Wait()
    0

SIGTERM on mono

On mono, System.Runtime.Loader isn't available, you need to catch the specific signal.

1: 
2: 
3: 
4: 
5: 
6: 
7: 
    let signals = 
        [|
            new Mono.Unix.UnixSignal(
                Mono.Unix.Native.Signum.SIGTERM
            )
        |]
    Mono.Unix.UnixSignal.WaitAny (signals, -1) |> ignore

Signals? REALLY???

Yeah. You probably wish you could just block on Console.ReadLine. But you don't get a terminal attached, so there is no console.

You get one of two things, depending on how you write your application and which runtime:

  • It exits immediately, blowing right through that ReadLine.
  • It eats up an entire CPU core in a tight loop reading from a console that isn't there.

Registering services

Once you have a nice application, ready for systemd and Unix signals, you need to tell systemd about it:

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
#deploy to /etc/systemd/system
[Unit]
Description=Hello F# service

[Service]
Type=simple # "real" unix daemons will fork - use "forking"
User=dave
ExecStart=/usr/bin/dotnet /home/dave/src/openfs/simpleservice/bin/Debug/netcoreapp2.0/simpleservice.dll
Restart=always # it will restart however it stops
RestartSec=20s # if it restarts too fast, systemd will stop restarting, throttle it here.

Copy to /etc/systemd/system/fshello.service, then systemctl enable fshello.

We can find the process that is listening on 8080:

1: 
lsof -i tcp:8080

and send SIGTERM to tell it to exit

1: 
kill $PID

Service control

systemctl subcommands:

  • start - start the service now
  • restart - restart the service now
  • stop - stop the service now
  • enable - enable it to start automatically
  • disable - disable automatic start (doesn't actually stop it)
  • reload - reloads the configuration (not the service)
  • status - show the service status

Limitations of systemd

  • Use systemd if you package your services to deploy directly onto the operating system
    • Someone needs root access to set you up
  • PID tracking is not a health check. Just because it's running doesn't mean it's healthy.
  • Only keeps instance of a service running - it has no real means to scale your application up.
  • Not cluster aware.

Docker

Image - built from a recipe (a Dockerfile) and uploaded to a registry. Images contain filesystem layers and can be based off other images, meaning they share the same layers.

Container - a process started on top of a copy of an image's filesystem using cgroups to limit the memory, CPU, network, and disk for the process.

Dockerfile - a set of instructions for building an image. Each instruction is executed by creating a container from the base image (FROM) and applying each instruction. After each instruction, the a new image layer is saved, eventually (if no errors), creating a new image with multiple layers.

Docker daemon

Docker has a daemon for running containers - interact with it using docker run. Since Dockerfiles can contain a health check which is stored in image metadata, dockerd will perform those health checks, although they are only really used by Swarm.

Dockerfile

1: 
2: 
3: 
4: 
5: 
FROM microsoft/dotnet
ADD ./bin/Release/netcoreapp2.0/publish /app
EXPOSE 8080
CMD ["/usr/bin/dotnet", "/app/httpservice.dll"]
HEALTHCHECK --interval=5s CMD ["curl", "http://localhost:8080/healthcheck"]

docker run

Run a container, and you can also find that systemd plays a role here:

1: 
2: 
3: 
docker run -p 8080:8080 -d fsserver-mono:0.0.3

systemctl status

Kubernetes: Cluster Scheduling

  • How should this job run?
    • Always 3 instances
    • Up to 246 MiB RAM, up to 80% of a CPU core
  • Where should the instances run?
    • Prefer certain hosts in the cluster
    • Require certain resources (network port, persistent storage, etc.)

Cluster schedulers "schedule" a job to run on the worker nodes in the cluster, placing jobs based on requested resources and available, balanced capacity.

Minikube time!

Download minikube - a small Go executable that bootstraps a little Kubernetes environment using your local hypervisor.

Point your local docker client at the minikube registry to push images there.

1: 
2: 
3: 
4: 
export DOCKER_TLS_VERIFY="1"
export DOCKER_HOST="tcp://172.16.106.141:2376"
export DOCKER_CERT_PATH="/home/dave/src/openfs/certs"
export DOCKER_API_VERSION="1.23"

Now when you build images, you're doing it in minikube's registry.

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
12: 
13: 
14: 
15: 
16: 
17: 
18: 
19: 
20: 
21: 
22: 
23: 
apiVersion: apps/v1beta1
kind: Deployment
metadata:
name: fsharp-mono-deployment
spec:
replicas: 3
template:
    metadata:
    labels:
        app: fsserver-mono
    spec:
    containers:
    - name: fsserver-mono
        image: fsserver-mono
        imagePullPolicy: Never
        ports:
        - containerPort: 8080
        livenessProbe:
        httpGet:
            path: /healthcheck
            port: 8080
        initialDelaySeconds: 5
        timeoutSeconds: 1

Register the deployment

1: 
2: 
3: 
4: 
5: 
kubectl create -f fsserver-mono.yaml

kubectl expose deployment fsharp-mono-deployment

minikube service fsharp-mono-deployment

Do some damage

 1: 
 2: 
 3: 
 4: 
 5: 
 6: 
 7: 
 8: 
 9: 
10: 
11: 
kubectl get services

curl http://172.16.106.141:30262/healthcheck

curl -X POST http://172.16.106.141:30262/zombie

curl http://172.16.106.141:30262/info

curl http://172.16.106.141:30262/die

curl http://172.16.106.141:30262/info

Summary

  • Include healthcheck API's - HTTP if possible
  • Be aware of target runtime differences
  • Join FSharp Software Foundation

https://github.com/ninjarobot/openfsharp-resilient