Serverless Python: Managing Python Dependencies in AWS Lambda Layer

Usage:

 build_layer.sh <layer_name> <path/to/requirements/file> <python-version>

for example,

 scripts/build_layer.sh tensorflow requirements/tensorflow.txt 3.8

where requirements/tensorflow.txt contains:

tensorflow==2.4.0

build_layer.sh

#!/bin/bash
set -eo pipefail

function usage {
    file_name=$0
    echo "usage: ${file_name} <layer_name> <path/to/requirements.txt/file> <python_version>"
    exit 1
}

layer_name=${1}
requirements_file=${2}
python_version=${3}

if [[ -z "${requirements_file}" ]]; then
  requirements_file="requirements.txt"
fi

if [[ -z "${layer_name}" || ! -f "${requirements_file}" ]]; then
  usage
fi

layer_path=".layers/${layer_name}/python/lib/python${python_version}/site-packages"
rm -rf "$layer_path"

echo "Installing packages from ${requirements_file}"
docker run --rm \
           --user $UID \
           --volume "$PWD:/var/task" \
           --workdir /var/task \
           --env CV_PYPI_USERNAME \
           --env CV_PYPI_PASSWORD \
           "lambci/lambda:build-python${python_version}" \
            pip install -Ur "${requirements_file}" --target "${layer_path}"

echo "Removing unnecessary packages from ${layer_path}"
rm -rf "$layer_path"/{botocore,boto3}*

echo "Removing unnecessary files/folders from ${layer_path}"
find "$layer_path" -type f -name "*.pyc" -exec rm -rf {} +
find "$layer_path" -type d -name "__pycache__" -exec rm -rf {} +
find "$layer_path" -type d \( -name "tests" -o -name "test" \)  -exec rm -rf {} +

echo "Layer creation finished at ${layer_path}"