DEV Community

t.okazaki
t.okazaki

Posted on

Save the HTTP request body received by Go Gin to AWS Kinesis Firehose

I wanted to run Kinesis Firehose in Go, but I couldn't find any sample code.
Here is a simple example.

package main

import (
    "fmt"
    "encoding/json"
    "github.com/aws/aws-sdk-go/aws"
    "github.com/aws/aws-sdk-go/aws/awserr"
    "github.com/aws/aws-sdk-go/aws/session"
    "github.com/aws/aws-sdk-go/service/firehose"
    "github.com/gin-gonic/gin"
)

type EntryRecord struct {
    PostBodyField string `json:"entry"`
    Host string `json:"host"`
    RemoteAddr string `json:"remoteaddr"`
}

const (
    deliveryStreamName = "firehose-dest-bucket"
)

func handleEntry(c *gin.Context) {

    buf := make([]byte, 2048)
    n, _ := c.Request.Body.Read(buf)
    body_filed := string(buf[0:n])
    fmt.Println(body_filed)

    entry_record := EntryRecord{
        PostBodyField: body_filed,
        Host: c.Request.Host,
        RemoteAddr: c.Request.RemoteAddr,
    }

    firehose_svc := firehose.New(session.New(), aws.NewConfig().WithRegion("ap-northeast-1"))
    record := &firehose.Record{}
    json, _ := json.Marshal(entry_record)

    record_byte := append([]byte(json), []byte("\n")...)

    record.SetData(record_byte)

    _, err := firehose_svc.PutRecord(
        &firehose.PutRecordInput{
            DeliveryStreamName: aws.String(deliveryStreamName),
            Record:             record,
        },
    )
    if err != nil {
        if awsErr, ok := err.(awserr.Error); ok {
            print(awsErr.Message())
        }
    }
}

func main() {
    r := gin.Default()
    r.POST("/submit", handleEntry)
    r.Run() 
}
Enter fullscreen mode Exit fullscreen mode

Prerequisites

  • You have already set up Kinesis Firehose
  • You have already set up S3 bucket to store the request body data.

The following is the part that is sending data to Firehose.

&firehose.PutRecordInput{
            DeliveryStreamName: aws.String(deliveryStreamName),
            Record:             record,
        },
Enter fullscreen mode Exit fullscreen mode

The argument Record is a structure defined in the SDK.

    record := &firehose.Record{}
    json, _ := json.Marshal(entry_record)

    record_byte := append([]byte(json), []byte("\n")...)

    record.SetData(record_byte)
Enter fullscreen mode Exit fullscreen mode

Reference

AWS Go SDK

Top comments (0)