Spring Data MongoDB — CRUD, Aggregations, Views and Materialized Views

Basic Setup

Setup a single node MongoDB instance using docker-compose.yaml and bring it up using the docker-compose up -d command

version: '3.9'

services:
  mongo:
    image: mongo
    container_name: mongodb
    ports:
      - 27017:27017
    volumes:
      - ./mongo:/data/db
    environment:
      - MONGO_INITDB_ROOT_USERNAME=mongo
      - MONGO_INITDB_ROOT_PASSWORD=root

I’d recommand using the MongoDB Compass application as the GUI tool.

Add the following MongoDB dependencies in your pom.xml or use spring initializer and select the Spring Data MongoDB dependency:

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-mongodb</artifactId>
</dependency>

Add the required data source configuration details to your application.properties/yaml files:

spring.data.mongodb.authentication-database=admin #Pre-existing DB. For auth related stuff
spring.data.mongodb.username=mongo #See docker-compose.yml file
spring.data.mongodb.password=root  #See docker-compose.yml file
spring.data.mongodb.database=school-service #Name of your database
spring.data.mongodb.host=localhost
spring.data.mongodb.port=27017

Create your Entities and Repositories (Let’s create a school management project)

@ Document annotation simply marks a class as being a domain object that needs to be persisted to the database, along with allowing us to choose the name of the collection to be used.

@ Id marks this attribute as the primary key. If the value of the @ Id field is not null, it’s stored in the database as-is; otherwise, the converter will assume we want to store an ObjectId in the database (either ObjectId, String or BigInteger work).

@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class Address {
    //...Other fields
    private String city;
    private String state;
}

@Document(value = "institutes")
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class Institute {
    @Id
    private String id;
    private String name;
    private Address address;
}

@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class User {
    private String firstName;
    private String lastName;
    private Gender gender; //Enum - MALE, FEMALE
    private LocalDate dob;
    //Incremented using a cron that runs every day!
    private int age;
    private String email;
    private String phone;
    private Address address;
}

@Data
@NoArgsConstructor
@AllArgsConstructor
public class Hobby {
    private String hobby;
    //0 - 10
    private short experience;
}

@Document("students")
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class Student {
    @Id
    private String id;
    private User userDetails;
    private String instituteId;
    private List<Hobby> hobbies;
    //...Other fields
}

@Document(value = "teachers")
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class Teacher {
    @Id
    private String id;
    private User userDetails;
    private String instituteId;
}

@Document(value = "classes")
@Data
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class InstituteClass {
    @Id
    private String id;
    private String instituteId;
    private String classId;
    private String section;
    private short year;
    private List<String> studentIds;
    private String classTeacherId; //Homeroom Teacher
}

@Repository
public interface InstituteRepository extends MongoRepository<Institute, String> {
    //A normal spring data query method.
    Optional<List<Institute>> findByName(String name);
}
//+ Other repositories

Add @ EnableMongoRepositories to your application configuration. Spring will provide you with proxy repository implementations using which you can do almost all CRUD related stuff. Simply autowire these repositories and you are good to go:

User defined queries

You can create user defined queries (and specify prototype — the where clause and projection — the select clause)

Using the @Query annotation:

@Repository
public interface StudentRepository extends MongoRepository<Student, String> {

    // ?0 is a Positional parameter - will be replaced by id!
    // value -> Prototype and fields -> Projection!
    // Will only populate the id and hobbies fields in the student document!
    @Query(value = "{'id' : ?0}", fields = "{hobbies : 1}")
    Student findHobbiesById(String id);

    @Query(value = "{'userDetails.gender': ?0, 'userDetails.age': {'$lt': ?1}}")
    List<Student> findAllByGenderWithAgeLessThan(Gender gender, int age);

    @Query(value = "{'hobbies': {'$elemMatch': {'hobby': ?0, 'experience': {'$gt': ?1}}}}")
    List<Student> getStudentsWithHobbyExperienceGreaterThan(String hobby, int experience);

}

Using MongoTemplate:

Let’s try to re-create the above methods using MongoTemplate. You can simply auto-wire MongoTemplate in your repository classes and start using it:

import org.springframework.data.mongodb.core.MongoTemplate;
import org.springframework.data.mongodb.core.query.Criteria;
import org.springframework.data.mongodb.core.query.Query;
import org.springframework.data.mongodb.core.query.Update;
import org.springframework.stereotype.Repository;
...

@Repository
@Slf4j
public class CustomStudentRepository {

    private final MongoTemplate mongoTemplate;

    public CustomStudentRepository(MongoTemplate mongoTemplate) {
        log.info("Initialized mongodb template " + mongoTemplate);
        this.mongoTemplate = mongoTemplate;
    }

    public List<Student> findAllByGenderWithAgeLessThan(Gender gender, int age) {
        Query query = new Query();
        //query.allowSecondaryReads();
        //query.withReadPreference(...
        //query.withReadConcern(...

        Criteria criteria = Criteria.where("userDetails.gender").is(gender.name()).and("userDetails.age").lt(10);

        query.addCriteria(criteria);

        //mongoTemplate.find(Query, Entity.class);
        return mongoTemplate.find(query, Student.class);
    }


    public List<Student> getStudentsWithHobbyExperienceGreaterThan(String hobby, int experience) {
        Query query = new Query();
        Criteria criteria = Criteria.where("hobbies").elemMatch(
                Criteria.where("hobby").is(hobby)
                        .and("experience").gt(experience)
        );
        query.addCriteria(criteria);

        return mongoTemplate.find(query, Student.class);
    }

}

Updates

Let’s assume you have a cron that runs at midnight and finds out all of the students whose birthday is today. We now need to increment their age by 1. Here is how you can do it using MongoTemplate:

//Todo - Instead, use in and pass-in a list of student ids!
public void incrementAgeBy1(String studentId) {
    Query query = new Query();
    query.addCriteria(Criteria.where("id").is(studentId));

    Update update = new Update();
    //update.set()
    //update.unset()
    //update.addToSet()
    //update.push()
    //and more...
    update.inc("userDetails.age", 1);

    //mongoTemplate.updateFirst(Query, UpdateDefinition, Entity.class);
    mongoTemplate.updateFirst(query, update, Student.class);
}

Aggregations

Let’s try to do the following aggregations — Find gender count for an institute and Find gender count for all institutes:

Using the @Aggregation annotation:

import org.springframework.data.mongodb.repository.Aggregation;
...

@Repository
public interface StudentRepository extends MongoRepository<Student, String> {
    //...Other methods

    record StudentsByGenderForInstitute(String gender, long total) {}

    @Aggregation(pipeline = {
            "{'$match': {'instituteId': ?0}}",
            "{'$group': {'_id': '$userDetails.gender', 'total': {'$count':{}}}}",
            "{'$project': {'_id': 0, 'gender': '$_id', 'total': 1}}"
    })
    List<StudentsByGenderForInstitute> getStudentsByGenderForInstitute(String instituteId);

    @Data
    class StudentsByGender {
        private String instituteId;
        private String gender;
        private long total;
    }

    @Aggregation(pipeline = {
            "{'$group': {'_id': {'instituteId: '$instituteId', 'gender': '$userDetails.gender'}, 'total': {'$count': {}}}}",
            "{'$project': {'_id': 0, 'instituteId': '$_id.instituteId', 'gender': '$_id.gender', 'total': 1}}"
    })
    //Not working for some reason - throws a stack overflow error
    List<StudentsByGender> getStudentsByGender();

}

Let’s try to do the same aggregations using MongoTemplate:

import org.springframework.data.mongodb.core.aggregation.*;
...

@Repository
@Slf4j
public class CustomStudentRepository {

    private final MongoTemplate mongoTemplate;

    public CustomStudentRepository(MongoTemplate mongoTemplate) {
        log.info("Initialized mongodb template " + mongoTemplate);
        this.mongoTemplate = mongoTemplate;
    }
    //...Other methods    

    record StudentsByGenderForInstitute(String gender, long total) {}

    public List<StudentsByGenderForInstitute> getStudentsByGenderForInstitute(String instituteId) {
        MatchOperation match = Aggregation.match(Criteria.where("instituteId").is(instituteId));
        GroupOperation group = Aggregation.group("userDetails.gender").count().as("total");
        ProjectionOperation project = Aggregation.project()
                .andExclude("_id")
                .and("_id").as("gender")
                .andInclude("total");

        Aggregation aggregation = Aggregation.newAggregation(match, group, project);

        AggregationResults<StudentsByGenderForInstitute> results =
                mongoTemplate.aggregate(aggregation, Student.class, StudentsByGenderForInstitute.class);
        return results.getMappedResults();
    }

    record StudentsByGender(String instituteId, String gender, long total) {}

    public List<StudentsByGender> getStudentsByGender() {
        GroupOperation group = Aggregation.group("instituteId", "userDetails.gender").count().as("total");

        ProjectionOperation project = Aggregation.project().andExclude("_id")
                .and("_id.instituteId").as("instituteId")
                .and("_id.gender").as("gender")
                .andInclude("total");

        Aggregation aggregation = Aggregation.newAggregation(group, project);
        AggregationResults<StudentsByGender> results =
                mongoTemplate.aggregate(aggregation, Student.class, StudentsByGender.class);
        return results.getMappedResults().stream()
                .sorted(Comparator.comparing(result -> result.instituteId)).collect(Collectors.toList());
    }

}

Views

A MongoDB view is a read-only queryable object whose contents are defined by an aggregation pipeline on other collections or views. MongoDB does not persist the view contents to disk. A view’s content is computed on-demand when a client queries the view.

Standard views use the indexes of the underlying collection. As a result, you cannot create, drop or re-build indexes on a standard view directly, nor get a list of indexes on the view.

A view definition pipeline cannot include the $out or the $merge stage. This restriction also applies to embedded pipelines, such as pipelines used in $lookup or $facet stages.

You can create a view/materialised view on any aggregation pipeline, but it’ll make the most sense if you are able to join multiple collections! — $lookup.

Lookup

$lookup performs a left outer join to a collection in the same database to filter in documents from the “joined” collection for processing. The $lookup stage adds a new array field to each input document. The new array field contains the matching documents from the “joined” collection. The $lookup stage passes these reshaped documents to the next stage. You’ll probably end up using $unwind (Kind of like flatMap) with $lookup since the joined collections is added as an array field!

Example: Create a View of Students with Institute Details

Let’s start with something:

//Aggregation.lookup(from, localfield, foreignfield, as);   
LookupOperation lookup = Aggregation.lookup("institutes", "instituteId", "_id", "institute");    
AggregationResults<Object> results = mongoTemplate.aggregate(Aggregation.newAggregation(lookup), Student.class, Object.class);

The results are like this (notice the array):

Let’s add the $unwind stage

LookupOperation lookup = Aggregation.lookup("institutes", "instituteId", "_id", "institute");
UnwindOperation unwind = Aggregation.unwind("institute");
AggregationResults<Object> results = mongoTemplate.aggregate(Aggregation.newAggregation(lookup, unwind), Student.class, Object.class);

Let’s combine these operations and create our view:

record StudentWithInstituteDetails(String _id, User userDetails, List<Hobby> hobbies, Institute institute) {}

@PostConstruct
public void createStudentsWithInstituteDetailsView() {
    LookupOperation lookup = Aggregation.lookup("institutes", "instituteId", "_id", "institute");
    UnwindOperation unwind = Aggregation.unwind("institute");
    AggregationOperation[] aggregationOperations = {lookup, unwind};
    mongoTemplate.createView("students-with-institute-details", Student.class, aggregationOperations);
}

public List<StudentWithInstituteDetails> getStudentsWithInstituteDetails() {
    return mongoTemplate.find(new Query(), StudentWithInstituteDetails.class, "students-with-institute-details");
}

Materialized Views

An on-demand materialized view is a pre-computed aggregation pipeline result that is stored on and read from disk. On-demand materialized views are typically the results of a $merge or $out stage. Materialized views are therefore much more performant than standard views.

$merge is preferred but make sure you think about how your materialised view will get refreshed when data updates happen. You don’t want your entire data to be refreshed every time a single data update happens!

Example — Create a materialised view for the following aggregation — Group students by school and gender

Your initial run should cover all schools and all students, but when a student enrols or leaves a particular school, you want to rerun the pipeline for only that combination

Example:

Materialised View =
{_id: {school: A, gender: Female}, count: 50}
{_id: {school: A, gender: Male}, count: 55}
{_id: {school: B, gender: Female}, count: 60}
{_id: {school: B, gender: Male}, count: 40}
...

Eg. a new female student joins school A
You only want the first document to be updated! (_id!)

@PostConstruct
public void createSchoolGenderView() throws Exception {
    LookupOperation lookup = Aggregation.lookup("institutes", "instituteId", "_id", "institute");
    UnwindOperation unwind = Aggregation.unwind("institute");
    GroupOperation group = Aggregation.group("institute.name", "userDetails.gender").count().as("count");
    MergeOperation merge = Aggregation.merge().intoCollection("school_gender_view").whenMatched(MergeOperation.WhenDocumentsMatch.replaceDocument()).build();

    mongoTemplate.aggregate(
            Aggregation.newAggregation(lookup, unwind, group, merge),
            Student.class,
            Object.class
    );
}

But how do I refresh this? You could create something like this:

public void createSchoolGenderView(String instituteId, String gender) {
    MatchOperation match = null;
    if (StringUtils.hasLength(instituteId) && StringUtils.hasLength(gender)) {
        match = Aggregation.match(
                Criteria.where("instituteId").is(instituteId)
                        .and("userDetails.gender").is(gender)
        );
    }

    LookupOperation lookup = Aggregation.lookup("institutes", "instituteId", "_id", "institute");
    UnwindOperation unwind = Aggregation.unwind("institute");
    GroupOperation group = Aggregation.group("institute.name", "userDetails.gender").count().as("count");
    MergeOperation merge = Aggregation.merge().intoCollection("school_gender_view").whenMatched(MergeOperation.WhenDocumentsMatch.replaceDocument()).build();
    Aggregation aggregation = null;
    if (Objects.nonNull(match)) {
        aggregation = Aggregation.newAggregation(match, lookup, unwind, group, merge);
    } else {
        aggregation = Aggregation.newAggregation(lookup, unwind, group, merge);
    }

    mongoTemplate.aggregate(
            aggregation,
            Student.class,
            Object.class
    );

}