DEV Community: weakpixel

(Part 2) Hate YAML? Build your next tool with HCL!

weakpixel — Sun, 10 Apr 2022 14:28:25 +0000

This is the second part of my HCL series. You find the first part here (Part 1)

In the second post of my HCL series I want to extend our example with:

Cobra Commandline
Variables
Functions

Cobra

Cobra is my favorite library to build command-line tools.

We start off with the example program from the first post (source).

As I write before I want to introduce you to the Cobra command-line tool. In order to use it we have to add a new import:

import (
    "fmt"
    "os"

    "github.com/spf13/cobra"
    // ...

Next rename the main() function to newRunCommand() and refactor it to return a cobra.Command

func newRunCommand() *cobra.Command {
    // contains all variables given by the user with --var "key=value"
    vars := []string{}
    cmd := cobra.Command{
        Use: "run"
        Short: "Executes tasks",
        RunE: func(cmd *cobra.Command, args []string) error {
            config := &Config{}
            err := hclsimple.Decode("example.hcl", []byte(exampleHCL), nil, config)
            if err != nil {
                return err
            }
            for _, task := range config.Tasks {
                fmt.Printf("Task: %s\n", task.Name)
                for _, step := range task.Steps {
                    fmt.Printf("    Step: %s %s\n", step.Type, step.Name)
                    var runner Runner
                    switch step.Type {
                    case "mkdir":
                        runner = &MkdirStep{}
                    case "exec":
                        runner = &ExecStep{}
                    default:
                        return fmt.Errorf("unknown step type %q", step.Type)
                    }

                    diags := gohcl.DecodeBody(step.Remain, nil, runner)
                    if diags.HasErrors() {
                        return diags
                    }
                    err = runner.Run()
                    if err != nil {
                        return err
                    }
                }
            }

            return nil
        },
    }
    // Define an optional "var" flag for the commnd
    cmd.Flags().StringArrayVar(&vars, "var", nil, "Sets variable. Format <name>=<value>")
    return &cmd
}

The Use field describes the subcommand name. The Short field allows defining a short command description.
The RunE implements the execution of the (sub-)command. It contains our HCL parsing code. Since RunE allows us to
return an error we also have refactored the code to just return an error instead of using os.Exit(1).

After that we implement a new main function looking like:

func main() {
    root := cobra.Command{
        Use: "taskexec",
    }
    root.AddCommand(newRunCommand())
    err := root.Execute()
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
}

The root command is just an empty cobra.Command. To the root command we add our subcommand with root.AddCommand(newRunCommand()).

Let's try out what happens if we run our program:

go run main.go 
Usage:
  taskexec [command]

Available Commands:
  completion  Generate the autocompletion script for the specified shell
  help        Help about any command
  run         Executes tasks

Flags:
  -h, --help   help for taskexec

Let's try to show the help for the subcommand:

go run main.go run -h
Executes tasks

Usage:
  taskexec run [flags]

Flags:
  -h, --help              help for run
      --var stringArray   Sets variable. Format <name>=<value>

Great! Next, we want to make use of the variables. To use variables in our HCL config, we must learn about the hcl.EvalContext

EvalContext

The hcl.EvalContext allows as to define variables and functions

type EvalContext struct {
    Variables map[string]cty.Value
    Functions map[string]function.Function
}

For now, we focus on the variables. The Variables map allows us to define the variable name as key and as value a cty.Value. The cty.Value is part of the github.com/zclconf/go-cty/cty package. The package provides a dynamic type system.

You can read more about cty on the github project.

Let's come back to hcl.EvalContext. Where is this context struct actually used? In our example code we have two instances:

hclsimple.Decode("example.hcl", []byte(exampleHCL), 
    /*&hcl.EvalContext{}*/ nil, config)

and

diags := gohcl.DecodeBody(step.Remain,
     /*&hcl.EvalContext{}*/ nil, runner)

Variables

In our command we have defined a vars slice which contains the user-defined variables in the format:

--var "key=value" ...

So let's get started and create hcl.EvalContext and populate it with the vars parameters from the command line.

func newEvalContext(vars []string) (*hcl.EvalContext, error) {
    varMap := map[string]cty.Value{}
    for _, v := range vars {
        el := strings.Split(v, "=")
        if len(el) != 2 {
            return nil, fmt.Errorf("invalid format: %s", v)
        }
        varMap[el[0]] = cty.StringVal(el[1])
    }

    ctx := &hcl.EvalContext{}
    ctx.Variables = map[string]cty.Value{
        "var": cty.ObjectVal(varMap),
    }
    return ctx, nil
}

We use the newEvalContext() function in our subcommand to create the EvalContext and use the context in all places where we decode the HCL document:

// ...
RunE: func(cmd *cobra.Command, args []string) error {
    ctx, err := newEvalContext(vars)
    if err != nil {
        return err
    }
    config := &Config{}
    err = hclsimple.Decode("example.hcl", []byte(exampleHCL), ctx, config)
    // ...
    for _, task := range config.Tasks {
        fmt.Printf("Task: %s\n", task.Name)
        for _, step := range task.Steps {
            // ...
            diags := gohcl.DecodeBody(step.Remain, ctx, runner)
            // ...
        }
    }

    return nil
},
// ...

And finally, we change our exampleHCL to make use of variables:

exampleHCL = `
    task "first_task" {
        step "mkdir" "build_dir" {
            path = var.buildDir
        }
        step "exec" "list_build_dir" {
            command = "ls ${var.buildDir}"
        }
    }
`

Let's try to execute the command without defining the buildDir variable:

go run main.go run 
...
example.hcl:4,15-24: Unsupported attribute; This object does not have an attribute named "buildDir"., and 1 other diagnostic(s)
exit status 1

Good, it fails with a detailed error message.

Now we try to execute the command with the needed variable:

go run main.go run --var buildDir=./build
Task: first_task
    Step: mkdir build_dir
    Step: exec list_build_dir

And it works as expected!

You can see the full source code here

Functions

Next, we want to explore how e.g. Terraform provides these nice inline functions which makes life so much easier to deal with input variables.
It might not make much sense in our example but let's try to implement a function that converts all cased letters into uppercase:

helloValue = "${upper("hello")} World"

To implement a function we must add a new module to our import "github.com/zclconf/go-cty/cty/function".
We have to use the function.Spec struct to create with function.New our function implementation:

var upperFn = function.New(&function.Spec{
    // Define the required parameters.
    Params: []function.Parameter{
        {
            Name:             "str",
            Type:             cty.String,
            AllowDynamicType: true,
        },
    },
    // Define the return type
    Type: function.StaticReturnType(cty.String),
    // Function implementation:
    Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
        in := args[0].AsString()
        out := strings.ToUpper(in)
        return cty.StringVal(out), nil
    },
})

And last we add the new function to our EvalContext:

func newEvalContext(vars []string) (*hcl.EvalContext, error) {

    // ...

    ctx.Functions = map[string]function.Function{
        "upper": upperFn,
    }
    return ctx, nil
}

Update the exampleHCL to make use of our brand new define function:

exampleHCL = `
    task "first_task" {
        step "mkdir" "build_dir" {
            path = upper(var.buildDir)
        }
        step "exec" "list_build_dir" {
            command = "ls ${ upper(var.buildDir) }"
        }
    }
`

Add some debug output to our example Step execution (mkdir, exec) and run the program:

go run main.go run --var "buildDir=./build"

Task: first_task
    Step: mkdir build_dir
        Path:./build
    Step: exec list_build_dir
        Command: ls ./BUILD

and as expected we have an upper case build directory.

If you don't want to implement all the functions yourself or you need some inspiration to implement a function you find want you looking for here:

Resources

Resources:

Full Source Code

package main

import (
    "fmt"
    "os"
    "strings"

    "github.com/spf13/cobra"
    "github.com/zclconf/go-cty/cty"

    "github.com/hashicorp/hcl/v2"
    "github.com/hashicorp/hcl/v2/gohcl"
    "github.com/hashicorp/hcl/v2/hclsimple"

    "github.com/zclconf/go-cty/cty/function"
)

var (
    exampleHCL = `
        task "first_task" {
            step "mkdir" "build_dir" {
                path = upper(var.buildDir)
            }
            step "exec" "list_build_dir" {
                command = "ls ${ upper(var.buildDir) }"
            }
        }
    `
)

func main() {
    root := cobra.Command{
        Use: "taskexec",
    }
    root.AddCommand(newRunCommand())
    err := root.Execute()
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
}

func newRunCommand() *cobra.Command {
    vars := []string{}
    cmd := cobra.Command{
        Use:   "run",
        Short: "Executes tasks",
        RunE: func(cmd *cobra.Command, args []string) error {
            ctx, err := newEvalContext(vars)
            if err != nil {
                return err
            }
            config := &Config{}
            err = hclsimple.Decode("example.hcl", []byte(exampleHCL), ctx, config)
            if err != nil {
                return err
            }
            for _, task := range config.Tasks {
                fmt.Printf("Task: %s\n", task.Name)
                for _, step := range task.Steps {
                    fmt.Printf("    Step: %s %s\n", step.Type, step.Name)
                    var runner Runner
                    switch step.Type {
                    case "mkdir":
                        runner = &MkdirStep{}
                    case "exec":
                        runner = &ExecStep{}
                    default:
                        return fmt.Errorf("unknown step type %q", step.Type)
                    }

                    diags := gohcl.DecodeBody(step.Remain, ctx, runner)
                    if diags.HasErrors() {
                        return diags
                    }
                    err = runner.Run()
                    if err != nil {
                        return err
                    }
                }
            }

            return nil
        },
    }

    cmd.Flags().StringArrayVar(&vars, "var", nil, "Sets variable. Format <name>=<value>")

    return &cmd
}

func newEvalContext(vars []string) (*hcl.EvalContext, error) {
    varMap := map[string]cty.Value{}
    for _, v := range vars {
        el := strings.Split(v, "=")
        if len(el) != 2 {
            return nil, fmt.Errorf("invalid format: %s", v)
        }
        varMap[el[0]] = cty.StringVal(el[1])
    }

    ctx := &hcl.EvalContext{}
    ctx.Variables = map[string]cty.Value{
        "var": cty.ObjectVal(varMap),
    }
    ctx.Functions = map[string]function.Function{
        "upper": upperFn,
    }
    return ctx, nil
}

var upperFn = function.New(&function.Spec{
    // Define the required parameters.
    Params: []function.Parameter{
        {
            Name:             "str",
            Type:             cty.String,
            AllowDynamicType: true,
        },
    },
    // Define the return type
    Type: function.StaticReturnType(cty.String),
    // Function implementation:
    Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
        in := args[0].AsString()
        out := strings.ToUpper(in)
        return cty.StringVal(out), nil
    },
})

type Config struct {
    Tasks []*Task `hcl:"task,block"`
}
type Task struct {
    Name  string  `hcl:"name,label"`
    Steps []*Step `hcl:"step,block"`
}

type Step struct {
    Type   string   `hcl:"type,label"`
    Name   string   `hcl:"name,label"`
    Remain hcl.Body `hcl:",remain"`
}

type ExecStep struct {
    Command string `hcl:"command"`
}

func (s *ExecStep) Run() error {
    fmt.Println("\tCommand: " + s.Command)
    return nil
}

type MkdirStep struct {
    Path string `hcl:"path"`
}

func (s *MkdirStep) Run() error {
    fmt.Println("\tPath:" + s.Path)
    return nil
}

type Runner interface {
    Run() error
}

Hate YAML? Build your next tool with HCL!

weakpixel — Wed, 30 Mar 2022 16:24:37 +0000

In this post, I want to show how you can implement your own tool using the HCL format.
The HCL configuration format is used by all the amazing HasiCorp tools like Terraform, Vault, and Nomad.

Most modern applications and tools use YAML these days which is generally an easy-to-read format but can also be the cause of a lot of pain because of the white space sensitivity. HCL on the other hand provides an easy-to-read format with a clear structure and additional features like variable interpolation and inline function calls.

Let's start with a really simple example to parse HCL.

task "first_task" {
    exec "list_current_dir" {
        command = "ls ."
    }

    exec "list_var_dir" {
        command = "ls /var"
    }
}

To map the HCL to our structs we can use struct tags:

type Config struct {
    Tasks []*Task `hcl:"task,block"`
}
type Task struct {
    Name  string     `hcl:"name,label"`
    Steps []*ExecStep `hcl:"exec,block"`
}

type ExecStep struct {
    Name    string `hcl:"name,label"`
    Command string `hcl:"command"`
}

func (s *ExecStep) Run() error {
    return nil
}

And to decode the HCL we can use the decode function from the hclsimple package

import (
    "fmt"
    "os"
    "github.com/hashicorp/hcl/v2/hclsimple"
)

var (
    exampleHCL = `
        task "first_task" {
            exec "list_current_dir" {
                command = "ls ."
            }

            exec "list_var_dir" {
                command = "ls /var"
            }
        }
    `
)

func main() {
    config := &Config{}
    err := hclsimple.Decode("example.hcl", []byte(exampleHCL), nil, config)
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    for _, task := range config.Tasks {
        fmt.Printf("Task: %s\n", task.Name)
        for _, step := range task.Steps {
            fmt.Printf("    Step: %s %q\n", step.Name, step.Command)
            err = step.Run()
            if err != nil {
                fmt.Println(err)
                os.Exit(1)
            }
        }
    }
}

That was easy!

But what is if I want to support a different Step types? Let's say I want to support mkdir to easily create directories.

task "first_task" {
    mkdir "build_dir" {
        path = "./build"
    }
    exec "list_var_dir" {
        command = "ls /var"
    }
}

If I run our tool I get the following error:

example.hcl:3,4-9: Unsupported block type; Blocks of type "mkdir" are not expected here.

We could update our Task struct and add a block list for "mkdir":

type Task struct {
    Name      string       `hcl:"name,label"`
    ExecSteps []*ExecStep  `hcl:"exec,block"`
    MkdirStep []*MkdirStep `hcl:"mkdir,block"`
}

but obviously, we would lose the execution order since we have two separate lists. This is not going to work for us.

As an alternative solution we could change our configuration and make the Step type a label:

task "first_task" {
    step "mkdir" "build_dir" {
        path = "./build/"
    }
    step "exec" "list_build_dir" {
        command = "ls ./build/"
    }
}

Let's reflect the configuration change to our structs.

type Config struct {
    Tasks []*Task `hcl:"task,block"`
}

type Task struct {
    Name  string  `hcl:"name,label"`
    Steps []*Step `hcl:"step,block"`
}

type Step struct {
    Type   string   `hcl:"type,label"`
    Name   string   `hcl:"name,label"`
    Remain hcl.Body `hcl:",remain"`
}

type ExecStep struct {
    Command string `hcl:"command"`
}

func (s *ExecStep) Run() error {
    return nil
}

type MkdirStep struct {
    Path string `hcl:"path"`
}

func (s *MkdirStep) Run() error {
    return nil
}

As you can see we have added a new Step struct and use it in the Tasks struct instead of the ExecStep
The Step struct has an extra field called Remain. This field is required to capture all fields of the actual Step implementation.We will see later how we use the Remain field to decode the fields into our actual Step implementation.

Lastly, we add a new interface that allows us to run the Step implementation:

type Runner interface {
    Run() error
}

You can see above that all our Steps implement the Runner interface.

Now we have to adapt our parsing code:

import (
    "fmt"
    "os"

    "github.com/hashicorp/hcl/v2"
    "github.com/hashicorp/hcl/v2/gohcl"
    "github.com/hashicorp/hcl/v2/hclsimple"
)

var (
    exampleHCL = `
        task "first_task" {
            step "mkdir" "build_dir" {
                path = "./build/"
            }
            step "exec" "list_build_dir" {
                command = "ls ./build/"
            }
        }
    `
)

func main() {
    config := &Config{}
    err := hclsimple.Decode("example.hcl", []byte(exampleHCL), nil, config)
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    for _, task := range config.Tasks {
        fmt.Printf("Task: %s\n", task.Name)
        for _, step := range task.Steps {
            fmt.Printf("    Step: %s %s\n", step.Type, step.Name)
            // The actual step implementation 
            // which implements the Runner interface
            var runner Runner

            // Determine the step implementation
            switch step.Type {
            case "mkdir":
                runner = &MkdirStep{}
            case "exec":
                runner = &ExecStep{}
            default:
                fmt.Printf("Unknown step type %q\n", step.Type)
                os.Exit(1)
            }

            // Decode remaining fields into our step implementation.
            diags := gohcl.DecodeBody(step.Remain, nil, runner)
            if diags.HasErrors() {
                fmt.Println(diags)
                os.Exit(1)
            }

            err = runner.Run()
            if err != nil {
                fmt.Println(err)
                os.Exit(1)
            }
        }
    }
}

The parsing does determine the Step type in the switch statement and creates an instance of the Struct. The struct is then the target of gohcl.DecodeBody(step.Remain, nil, runner) which decodes the remaining fields.

Voilà, we have an easy-to-extend Task execution engine.

Resources

Godocs:

Others:

buildmedia.readthedocs.org/hcl.pdf

Source Gists:

Full Source Code

package main

import (
    "fmt"
    "os"

    "github.com/hashicorp/hcl/v2"
    "github.com/hashicorp/hcl/v2/gohcl"
    "github.com/hashicorp/hcl/v2/hclsimple"
)

var (
    exampleHCL = `
        task "first_task" {
            step "mkdir" "build_dir" {
                path = "./build/"
            }
            step "exec" "list_build_dir" {
                command = "ls ./build/"
            }
        }
    `
)

func main() {
    config := &Config{}
    err := hclsimple.Decode("example.hcl", []byte(exampleHCL), nil, config)
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }
    for _, task := range config.Tasks {
        fmt.Printf("Task: %s\n", task.Name)
        for _, step := range task.Steps {
            fmt.Printf("    Step: %s %s\n", step.Type, step.Name)

            var runner Runner

            switch step.Type {
            case "mkdir":
                runner = &MkdirStep{}
            case "exec":
                runner = &ExecStep{}
            default:
                fmt.Printf("Unknown step type %q\n", step.Type)
                os.Exit(1)
            }

            diags := gohcl.DecodeBody(step.Remain, nil, runner)
            if diags.HasErrors() {
                fmt.Println(diags)
                os.Exit(1)
            }
            err = runner.Run()
            if err != nil {
                fmt.Println(err)
                os.Exit(1)
            }
        }
    }
}

type Config struct {
    Tasks []*Task `hcl:"task,block"`
}
type Task struct {
    Name  string  `hcl:"name,label"`
    Steps []*Step `hcl:"step,block"`
}

type Step struct {
    Type   string   `hcl:"type,label"`
    Name   string   `hcl:"name,label"`
    Remain hcl.Body `hcl:",remain"`
}

type ExecStep struct {
    Command string `hcl:"command"`
}

func (s *ExecStep) Run() error {
    // Implement me
    return nil
}

type MkdirStep struct {
    Path string `hcl:"path"`
}

func (s *MkdirStep) Run() error {
    // Implement me
    return nil
}

type Runner interface {
    Run() error
}