Semantic search on go + python

stack:

golang (gorm, pgvector-go)
python (langchain)
docker (docker compose)

ref:

setup pgvector

using docker-compose to setup postgres + pgvector

version: '2'
services:
  db-postgres:
    image: postgres
    restart: always
    user: postgres_user
    environment:
      POSTGRES_USER: postgres_user
      POSTGRES_PASSWORD: postgres_pass
    volumes:
      - db-data:/var/lib/postgresql/data
    healthcheck:
      test: [ "CMD", "pg_isready" ]
      interval: 10s
      timeout: 5s
      retries: 5
    ports:
      - "5432:5432"
    networks:
      - default
volumes:
  db-data:
networks:
  default:
    name: wawancara_namespace

separate embedding data and the content

package models

import (
    "time"

    "github.com/pgvector/pgvector-go"
)

// A) Core item
type Item struct {
    ID          int           `gorm:"primaryKey"`
    Name        string        `gorm:"not null"`
    Description string        `gorm:"type:text"`
}

// B) Embeddings table
type ItemVector struct {
    ID         int              `gorm:"primaryKey"`
    ItemID     int              `gorm:"not null;index"`
    Embedding  pgvector.Vector  `gorm:"type:vector(1536);not null"`
    Item       *Item
}

Generate embed data

as per langchain documentation about semantic search, i only implement splitter and embedding method

def text_embed(content: str) -> List[List[np.float32]]:
    # 1) Split into chunks
    splitter = RecursiveCharacterTextSplitter(
        chunk_size=1000,
        chunk_overlap=200,
        add_start_index=True,
    )
    chunks = splitter.split_text(content)

    # 2) Embed each chunk
    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

    all_vectors: List[List[float]] = []
    for chunk in chunks:
        # 3a) Get a 64‑bit embedding from OpenAI
        vec64: List[float] = embeddings.embed_query(chunk)
        # 3b) Cast down to 32‑bit and convert to plain Python list
        vec32 = np.array(vec64, dtype=np.float32).tolist()
        all_vectors.append(vec32)

    return all_vectors

need to make it to float32 here, because pgvector only accept float32

save data to pgvector

func SaveChunkEmbeddings(db *gorm.DB, cvRankerID int, allVectors [][]float32) error {
    // Bulk insert version:
    items := make([]models.ItemVector, len(allVectors))
    for i, vec := range allVectors {
        items[i] = models.ItemVector{
            ItemID:     cvRankerID,
            ChunkIndex: i,
            Embedding:  pgvector.NewVector(vec),
        }
    }

    if err := db.CreateInBatches(items, 100).Error; err != nil {
        log.Printf("bulk insert embeddings failed: %v", err)
        return err
    }
    return nil
}

now we have data vector for our item

Get embed data from query

When user search something, we just need to get vector of the query

def query_embed(content: str) -> List[np.float32]:
    """
    Embed a single query string into a 32-bit vector.
    """
    embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
    vec64: List[float] = embeddings.embed_query(content)
    vec32 = np.array(vec64, dtype=np.float32).tolist()
    return vec32

then use the vector in our vector table search

func Top5ItemsByEmbedding(db *gorm.DB, embedding []float32) ([]ItemVector, error) {
    var results []ItemVector

    vec := pgvector.NewVector(embedding)

    err := db.
        Model(&entity.ItemVector{}).
Join("Item")
        Select("item_id, MIN(embedding <-> ?) AS score", vec).
        Group("item_id").
        Order("score ASC").
        Limit(5).
        Scan(&results).Error

    return results, err
}

DEV Community