May 30, 2019

Elixir resources
to help with transition to Elixir

Updated 2019-10-08

We are planning introducing Elixir into our toolbox. This page summarizes key resources we have user / are using for learning Elixir and pushing it to production. Feel free to propose changes via pull-request.

Deployment & Containers/Kubernetes

Motivation is to be able to deploy apps leveraging OTP to k8s (and running in containers). Especially important piece of having a support for OTP is to be able to use things like long-running GenServer processes, migrate state etc. Valuable resources for this topic are

This leads into the following key building blocks:

Distributed systems / data-types

  • Using Rust to Scale Elixir for 11 Million Concurrent Users
    • Rust implementation of SortedSet which is then used by Elixir backend
  • An Adventure in Distributed Programming by Wiebe-Marten Wijnja
  • Distributing Phoenix -- Part 2: Learn You a 𝛿-CRDT for Great Good
  • Building Resilient Systems with Stacking by Chris Keathley
    • Recording from ElixrConf EU 2019
      • Overview of techniques which helps in building more resilient systems. Refers to How Complex Systems Fail for parallels between medical systems and complex distributed services.

      • Circuit brakers: Recommended implementation is fuse.

      • Configuration: Should avoid use of "mix configs", instead he pointed to (his) project Vapor. Example of usage (from the talk, chech project for other one):

        defmodule Jenga.Application do
          use Application
        
          def start(_type, _args) do
            config = [
                port: "PORT",
                db_url: "DB_URL",
            ]
        
            children = [
                {Jenga.Config, config},
            ]
        
            opts = [strategy: :one_for_one, name: Jenga.Supervisor]
            Supervisor.start_link(children, opts)
          end
        end
        
        defmodule Jenga.Config do
            use GenServer
        
            def start_link(desired_config) do
                GenServer.start_link(__MODULE__, desired_config, name: __MODULE__)
            end
        
            def init(desired) do
                :jenga_config = :ets.new(:jenga_config, [:set, :protected, :named_table])
                case load_config(:jenga_config, desired) do
                :ok ->
                    {:ok, %{table: :jenga_config, desired: desired}}
                :error ->
                    {:stop, :could_not_load_config}
                end
            end
        
            defp load_config(table, config, retry_count \\ 0)
            defp load_config(_table, [], _), do: :ok
            defp load_config(_table, _, 10), do: :error
            defp load_config(table, [{k, v} | tail], retry_count) do
                case System.get_env(v) do
                nil ->
                    load_config(table, [{k, v} | tail], retry_count + 1)
                value ->
                    :ets.insert(table, {k, value})
                    load_config(table, tail, retry_count)
                end
            end
        end
      • Monitoring: you can use Erlang's alarms. Example from the talk, which takes database as dependency and if not reachable will raise an alarm:

        defmodule Jenga.Database.Watchdog do
        use GenServer
        
        def init(:ok) do
            schedule_check()
            {:ok, %{status: :degraded, passing_checks: 0}}
        end
        
        def handle_info(:check_db, state) do
            status = Jenga.Database.check_status()
            state = change_state(status, state)
            schedule_check()
            {:noreply, state}
        end
        
        defp change_state(result, %{status: status, passing_checks: count}) do
            case {result, status, count} do
            {:ok, :connected, count} ->
                if count == 3 do
                :alarm_handler.clear_alarm(@alarm_id)
                end
                %{status: :connected, passing_checks: count + 1}
        
            {:ok, :degraded, _} ->
                %{status: :connected, passing_checks: 0}
        
            {:error, :connected, _} ->
                :alarm_handler.set_alarm({@alarm_id, "We cannot connect to the database”})
                %{status: :degraded, passing_checks: 0}
                {:error, :degraded, _} ->
                    %{status: :degraded, passing_checks: 0}
            end
        end
        end

        Then alarm handle can be added:

        defmodule Jenga.Application do
          use Application
        
          def start(_type, _args) do
            config = [
                port: "PORT",
                db_url: "DB_URL",
            ]
        
            :gen_event.swap_handler(
                :alarm_handler,
                {:alarm_handler, :swap},
                {Jenga.AlarmHandler, :ok}
            )
        
            children = [
                {Jenga.Config, config},
                Jenga.Database.Supervisor,
            ]
        
            opts = [strategy: :one_for_one, name: Jenga.Supervisor]
            Supervisor.start_link(children, opts)
          end
        end
        
        defmodule Jenga.AlarmHandler do
          require Logger
        
          def init({:ok, {:alarm_handler, _old_alarms}}) do
            Logger.info("Installing alarm handler")
            {:ok %{}}
          end
        
          def handle_event({:set_alarm, :database_disconnected}, alarms) do
            # Do something with the alarm rising (e.g. notify monitoring)
            Logger.error("Database connection lost")
            {:ok, alarms}
          end
        
          def handle_event({:clear_alarm, :database_disconnected}, alarms) do
            # Do something with the alarm being cleared (e.g. notify monitoring)
            Logger.error("Database connection recovered")
            {:ok, alarms}
          end
        
          def handle_event(event, state) do
            Logger.info("Unhandled alarm event: #{inspect(event)}")
            {:ok, state}
          end
        end

Best practices

Code/Development

  • typical suspect - credo

    mix credo --strict
  • spend time writing documentation in code with ExDoc

    • write typespecs as they are pulled by ExDoc but also used by tools like dialyzer
    • deploy dialyzer from the very beginning of the project
  • use official formatter in your projects

    mix format --check-formatted
  • nice writeup about putting these tools together - https://itnext.io/enforcing-code-quality-in-elixir-20f87efc7e66

Code Design

Ops/Infrastructure/Monitoring