Please proceed here for a new version of the article. The method described below doesn't work anymore (and it's not really good either).
In videogames, we often need to show the player beautiful cinematics, either for ending or in the middle of the playthrough. To do it, we need to somehow load video frames and render them on the screen. Here's how to do that using Pixel game library and goav, Golang bindings for FFmpeg.
Initial setup
First we need to create a GLFW window for rendering OpenGL stuff:
package main
import (
"fmt"
"github.com/faiface/pixel"
"github.com/faiface/pixel/pixelgl"
colors "golang.org/x/image/colornames"
)
const (
// WindowWidth is the width of the window.
WindowWidth = 1280
// WindowHeight is the height of the window.
WindowHeight = 720
)
func run() {
// Create a new window.
cfg := pixelgl.WindowConfig{
Title: "Pixel Rocks!",
Bounds: pixel.R(0, 0,
float64(WindowWidth), float64(WindowHeight)),
VSync: false,
}
win, err := pixelgl.NewWindow(cfg)
handleError(err)
fps := 0
perSecond := time.Tick(time.Second)
for !win.Closed() {
win.Clear(colors.White)
win.Update()
// Show FPS in the window title.
fps++
select {
case <-perSecond:
win.SetTitle(fmt.Sprintf("%s | FPS: %d",
cfg.Title, fps))
fps = 0
default:
}
}
}
func main() {
pixelgl.Run(run)
}
func handleError(err error) {
if err != nil {
panic(err)
}
}
Obtaining video frames
Now we need to obtain frames from the video stream of the file. goav provides an example code on how to do that. To make it work with Pixel game library, we need to set the frame decoding format to avcodec.AV_PIX_FMT_RGBA which is used by Pixel.
We also need a way to send the decoded frames to the renderer. For this task we will use a thread-safe frame buffer channel. First we should specify its size:
const (
FrameBufferSize = 1024
)
The greater the buffer size, the faster the frame transfer from the decoder to the renderer. Now to the channel creation:
frameBuffer := make(chan *pixel.PictureData, FrameBufferSize)
When the frame transfer is complete, we need to close the channel, but we also have to make sure the renderer got all the frames from the buffer. So here's the code for closing the frame buffer:
go func() {
for {
if len(frameBuffer) <= 0 {
close(frameBuffer)
break
}
}
}()
The complete code for reading video frames is presented below:
func readVideoFrames(videoPath string) <-chan *pixel.PictureData {
// Create a frame buffer.
frameBuffer := make(chan *pixel.PictureData, FrameBufferSize)
go func() {
// Open a video file.
pFormatContext := avformat.AvformatAllocContext()
if avformat.AvformatOpenInput(&pFormatContext, videoPath, nil, nil) != 0 {
fmt.Printf("Unable to open file %s\n", videoPath)
os.Exit(1)
}
// Retrieve the stream information.
if pFormatContext.AvformatFindStreamInfo(nil) < 0 {
fmt.Println("Couldn't find stream information")
os.Exit(1)
}
// Dump information about the video to stderr.
pFormatContext.AvDumpFormat(0, videoPath, 0)
// Find the first video stream
for i := 0; i < int(pFormatContext.NbStreams()); i++ {
switch pFormatContext.Streams()[i].
CodecParameters().AvCodecGetType() {
case avformat.AVMEDIA_TYPE_VIDEO:
// Get a pointer to the codec context for the video stream
pCodecCtxOrig := pFormatContext.Streams()[i].Codec()
// Find the decoder for the video stream
pCodec := avcodec.AvcodecFindDecoder(avcodec.
CodecId(pCodecCtxOrig.GetCodecId()))
if pCodec == nil {
fmt.Println("Unsupported codec!")
os.Exit(1)
}
// Copy context
pCodecCtx := pCodec.AvcodecAllocContext3()
if pCodecCtx.AvcodecCopyContext((*avcodec.
Context)(unsafe.Pointer(pCodecCtxOrig))) != 0 {
fmt.Println("Couldn't copy codec context")
os.Exit(1)
}
// Open codec
if pCodecCtx.AvcodecOpen2(pCodec, nil) < 0 {
fmt.Println("Could not open codec")
os.Exit(1)
}
// Allocate video frame
pFrame := avutil.AvFrameAlloc()
// Allocate an AVFrame structure
pFrameRGB := avutil.AvFrameAlloc()
if pFrameRGB == nil {
fmt.Println("Unable to allocate RGB Frame")
os.Exit(1)
}
// Determine required buffer size and allocate buffer
numBytes := uintptr(avcodec.AvpictureGetSize(
avcodec.AV_PIX_FMT_RGBA, pCodecCtx.Width(),
pCodecCtx.Height()))
buffer := avutil.AvMalloc(numBytes)
// Assign appropriate parts of buffer to image planes in pFrameRGB
// Note that pFrameRGB is an AVFrame, but AVFrame is a superset
// of AVPicture
avp := (*avcodec.Picture)(unsafe.Pointer(pFrameRGB))
avp.AvpictureFill((*uint8)(buffer),
avcodec.AV_PIX_FMT_RGBA, pCodecCtx.Width(), pCodecCtx.Height())
// initialize SWS context for software scaling
swsCtx := swscale.SwsGetcontext(
pCodecCtx.Width(),
pCodecCtx.Height(),
(swscale.PixelFormat)(pCodecCtx.PixFmt()),
pCodecCtx.Width(),
pCodecCtx.Height(),
avcodec.AV_PIX_FMT_RGBA,
avcodec.SWS_BILINEAR,
nil,
nil,
nil,
)
// Read frames and save first five frames to disk
packet := avcodec.AvPacketAlloc()
for pFormatContext.AvReadFrame(packet) >= 0 {
// Is this a packet from the video stream?
if packet.StreamIndex() == i {
// Decode video frame
response := pCodecCtx.AvcodecSendPacket(packet)
if response < 0 {
fmt.Printf("Error while sending a packet to the decoder: %s\n",
avutil.ErrorFromCode(response))
}
for response >= 0 {
response = pCodecCtx.AvcodecReceiveFrame(
(*avcodec.Frame)(unsafe.Pointer(pFrame)))
if response == avutil.AvErrorEAGAIN ||
response == avutil.AvErrorEOF {
break
} else if response < 0 {
//fmt.Printf("Error while receiving a frame from the decoder: %s\n",
//avutil.ErrorFromCode(response))
//return
}
// Convert the image from its native format to RGB
swscale.SwsScale2(swsCtx, avutil.Data(pFrame),
avutil.Linesize(pFrame), 0, pCodecCtx.Height(),
avutil.Data(pFrameRGB), avutil.Linesize(pFrameRGB))
// Save the frame to the frame buffer.
frame := getFrameRGBA(pFrameRGB,
pCodecCtx.Width(), pCodecCtx.Height())
frameBuffer <- frame
}
}
// Free the packet that was allocated by av_read_frame
packet.AvFreePacket()
}
go func() {
for {
if len(frameBuffer) <= 0 {
close(frameBuffer)
break
}
}
}()
// Free the RGB image
avutil.AvFree(buffer)
avutil.AvFrameFree(pFrameRGB)
// Free the YUV frame
avutil.AvFrameFree(pFrame)
// Close the codecs
pCodecCtx.AvcodecClose()
(*avcodec.Context)(unsafe.Pointer(pCodecCtxOrig)).AvcodecClose()
// Close the video file
pFormatContext.AvformatCloseInput()
// Stop after saving frames of first video straem
break
default:
fmt.Println("Didn't find a video stream")
os.Exit(1)
}
}
}()
return frameBuffer
}
We allocate a separate goroutine for the frame decoder so it can do its work in its own pace and just put all the results in the frame buffer.
Converting video frames
Now we need to bring the extracted video frame to the form appropriate for Pixel. First we should extract raw RGBA bytes:
func getFrameRGBA(frame *avutil.Frame, width, height int) *pixel.PictureData {
pix := []byte{}
for y := 0; y < height; y++ {
data0 := avutil.Data(frame)[0]
buf := make([]byte, width*4)
startPos := uintptr(unsafe.Pointer(data0)) +
uintptr(y)*uintptr(avutil.Linesize(frame)[0])
for i := 0; i < width*4; i++ {
element := *(*uint8)(unsafe.Pointer(startPos + uintptr(i)))
buf[i] = element
}
pix = append(pix, buf...)
}
return pixToPictureData(pix, width, height)
}
To create *pixel.PictureData out of these bytes, we'll use this simple code:
func pixToPictureData(pixels []byte, width, height int) *pixel.PictureData {
picData := pixel.MakePictureData(pixel.
R(0, 0, float64(width), float64(height)))
for y := height - 1; y >= 0; y-- {
for x := 0; x < width; x++ {
picData.Pix[(height-y-1)*width+x].R = pixels[y*width*4+x*4+0]
picData.Pix[(height-y-1)*width+x].G = pixels[y*width*4+x*4+1]
picData.Pix[(height-y-1)*width+x].B = pixels[y*width*4+x*4+2]
picData.Pix[(height-y-1)*width+x].A = pixels[y*width*4+x*4+3]
}
}
return picData
}
We need to fill the Pix array vice versa because this is how Pixel treats picture data.
Rendering video frames
Now let's create a new animated sprite to output the video frames:
videoSprite := pixel.NewSprite(nil, pixel.Rect{})
videoTransform := pixel.IM.Moved(pixel.V(
float64(WindowWidth)/2, float64(WindowHeight)/2))
frameBuffer := readVideoFrames(os.Args[1])
The path to the video file is specified as a command line argument.
Then let's start rendering the video frames:
select {
case frame, ok: = <-frameBuffer:
if !ok {
os.Exit(0)
}
if frame != nil {
videoSprite.Set(frame, frame.Rect)
}
default:
}
videoSprite.Draw(win, videoTransform)
Here's the result:
This method can be adapted for other Golang game engines like Ebiten if you know the way to convert RGBA bytes to the appropriate form.
Enjoy the full source code.
Top comments (2)
but it no sound
I wrote a new article on this topic. It tells how to play not only video but also sound from media files. medium.com/@maximgradan/playing-vi...