In this article, we will build an EC2 inventory for all the accounts and all the regions in AWS Organization. We will provision the lambda, associated roles and S3 bucket through CloudFormation. We will use Python 3.7 in lambda.
Our first two tasks will be to get the list of accounts in AWS Organization and to get the list of regions. We will get the lists through boto3 clients.
import boto3
def get_account_list(client):
account_info = client.list_accounts()
account_list = account_info.get('Accounts')
next_token = account_info.get('NextToken')
while next_token != None:
account_info = client.list_accounts(NextToken=next_token)
account_list.extend(account_info.get('Accounts'))
next_token = account_info.get('NextToken')
return account_list
def get_region_list(client):
regions = [region['RegionName']
for region in client.describe_regions()['Regions']]
return regions
def lambda_handler(event, context):
client = boto3.client('organizations')
account_list = get_account_list(client)
client = boto3.client('ec2')
region_list = get_region_list(client)
Boto3 clients use pagination by default in response to ensure that the operation returns quickly and successfully. It provides a NextToken to fetch the next page of the response. So, it is not necessary that all the results are there in the response. So, it is a good practice to always iterate the request by NextToken until NexToken == None. We have also listed the regions using describe_regions() function of boto3 EC2 client.
Next, we try to fetch the EC2 instances from the current account only.
import boto3
def get_instance_list_from_response(response):
instance_list = []
for reservation in response.get('Reservations'):
if 'Instances' in reservation:
instance_list.extend(reservation.get('Instances'))
return instance_list
def get_ec2_list(client):
response = client.describe_instances()
ec2_list = get_instance_list_from_response(response)
next_token = response.get('NextToken')
while next_token != None:
response = client.describe_instances(
NextToken=next_token
)
ec2_list.extend(get_instance_list_from_response(response))
next_token = response.get('NextToken')
return ec2_list
def lambda_handler(event, context):
client = boto3.client('ec2')
region_list = get_region_list(client)
ec2_list = get_ec2_list(client)
In this list, it contains many information about the ec2 instances (https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2/client/describe_instances.html).So, we can filter out the information we need.
So, we can filter out the information we need.
def get_ec2_from_raw(
client, raw_ec2_instances, account_name
):
ec2_instances = []
for raw_ec2_instance in raw_ec2_instances:
ec2_instance = {}
ec2_instance['Account Name'] = account_name
ec2_instance['Region'] = (
raw_ec2_instance.get('Placement').get('AvailabilityZone')[:-1]
)
ec2_instance['Instance Id'] = (raw_ec2_instance.get(
'InstanceId'
))
ec2_instance['Instance Type'] = (raw_ec2_instance.get(
'InstanceType'
))
ec2_instance['Platform'] = (raw_ec2_instance.get(
'Platform'
))
ec2_instance['Private Ip Address'] = (raw_ec2_instance.get(
'PrivateIpAddress'
))
ec2_instance['Public Ip Address'] = (raw_ec2_instance.get(
'PublicIpAddress'
))
ec2_instance['Vpc Id'] = (raw_ec2_instance.get(
'VpcId'
))
ec2_instance['Subnet Id'] = (raw_ec2_instance.get(
'SubnetId'
))
ec2_instances.append(ec2_instance)
return ec2_instances
Now, if you remember, only getting the ec2 instances from current account is not enough for us. We have to collect the EC2 instances from all the accounts in the AWS Organization and and all the regions. So, we have to merge the above code blocks together. We will also add code to store the result in S3.
import boto3
def get_account_list(client):
account_info = client.list_accounts()
account_list = account_info.get('Accounts')
next_token = account_info.get('NextToken')
while next_token != None:
account_info = client.list_accounts(NextToken=next_token)
account_list.extend(account_info.get('Accounts'))
next_token = account_info.get('NextToken')
return account_list
def get_region_list(client):
regions = [region['RegionName']
for region in client.describe_regions()['Regions']]
return regions
def get_instance_list_from_response(response):
instance_list = []
for reservation in response.get('Reservations'):
if 'Instances' in reservation:
instance_list.extend(reservation.get('Instances'))
return instance_list
def get_ec2_list(client):
response = client.describe_instances()
ec2_list = get_instance_list_from_response(response)
next_token = response.get('NextToken')
while next_token != None:
response = client.describe_instances(
NextToken=next_token
)
ec2_list.extend(get_instance_list_from_response(response))
next_token = response.get('NextToken')
return ec2_list
def get_ec2_from_raw(
client, raw_ec2_instances, account_name
):
ec2_instances = []
for raw_ec2_instance in raw_ec2_instances:
ec2_instance = {}
ec2_instance['Account Name'] = account_name
ec2_instance['Region'] = (
raw_ec2_instance.get('Placement').get('AvailabilityZone')[:-1]
)
ec2_instance['Instance Id'] = (raw_ec2_instance.get(
'InstanceId'
))
ec2_instance['Instance Type'] = (raw_ec2_instance.get(
'InstanceType'
))
ec2_instance['Platform'] = (raw_ec2_instance.get(
'Platform'
))
ec2_instance['Private Ip Address'] = (raw_ec2_instance.get(
'PrivateIpAddress'
))
ec2_instance['Public Ip Address'] = (raw_ec2_instance.get(
'PublicIpAddress'
))
ec2_instance['Vpc Id'] = (raw_ec2_instance.get(
'VpcId'
))
ec2_instance['Subnet Id'] = (raw_ec2_instance.get(
'SubnetId'
))
ec2_instances.append(ec2_instance)
return ec2_instances
def role_arn_to_session(
role_arn, role_session_name, region
):
client = boto3.client("sts")
response = client.assume_role(
RoleArn=role_arn,
RoleSessionName=role_session_name
)
return boto3.Session(
aws_access_key_id=response.get("Credentials").get("AccessKeyId"),
aws_secret_access_key=response.get("Credentials").get("SecretAccessKey"),
aws_session_token=response.get("Credentials").get("SessionToken"),
region_name=region
)
def get_boto3_session(event, account_id, region):
return role_arn_to_session(
role_arn="arn:aws:iam::" + account_id + ":role/" + event.get(
'organization_access_role', 'OrganizationAccountAccessRole'
),
role_session_name=f"{account_id}-crossaccount-role",
region=region
)
def get_ec2_instances(event, account, region_list):
ec2_instances_in_account = []
if account.get('Status') == 'ACTIVE':
for region in region_list:
session = get_boto3_session(event, account.get('Id'), region)
client = session.client('ec2')
raw_ec2_instances = get_ec2_list(client)
ec2_instances_in_region = get_ec2_from_raw(
client, raw_ec2_instances, account.get('Name')
)
ec2_instances_in_account.extend(ec2_instances_in_region)
return ec2_instances_in_account
def lambda_handler(event, context):
client = boto3.client('organizations')
account_list = get_account_list(client)
client = boto3.client('ec2')
region_list = get_region_list(client)
ec2_instances_for_s3 = []
for account in account_list:
ec2_instances = get_ec2_instances(
event, account, region_list
)
ec2_instances_for_s3.extend(ec2_instances)
In get_ec2_instances function, we are creating boto3 session with particular account ID’s and regions from our account list and region list. We are assuming the organization access role in that account and getting Access Key, Secret Key and Session Token to create the session. After that, we are using clients from that session to get the ec2 list from that particular account and region.
To know more abour organization access role, please visit https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_accounts_access.html
Now, we will add functions to store the result in S3. We will store the result in S3 by converting our result to CSV string object. So, our final code will look like this.
import os
import boto3
import io
import csv
from datetime import datetime
def get_account_list(client):
account_info = client.list_accounts()
account_list = account_info.get('Accounts')
next_token = account_info.get('NextToken')
while next_token != None:
account_info = client.list_accounts(NextToken=next_token)
account_list.extend(account_info.get('Accounts'))
next_token = account_info.get('NextToken')
return account_list
def get_region_list(client):
regions = [region['RegionName']
for region in client.describe_regions()['Regions']]
return regions
def get_instance_list_from_response(response):
instance_list = []
for reservation in response.get('Reservations'):
if 'Instances' in reservation:
instance_list.extend(reservation.get('Instances'))
return instance_list
def get_ec2_list(client):
response = client.describe_instances()
ec2_list = get_instance_list_from_response(response)
next_token = response.get('NextToken')
while next_token != None:
response = client.describe_instances(
NextToken=next_token
)
ec2_list.extend(get_instance_list_from_response(response))
next_token = response.get('NextToken')
return ec2_list
def get_ec2_from_raw(
client, raw_ec2_instances, account_name
):
ec2_instances = []
for raw_ec2_instance in raw_ec2_instances:
ec2_instance = {}
ec2_instance['Account Name'] = account_name
ec2_instance['Region'] = (
raw_ec2_instance.get('Placement').get('AvailabilityZone')[:-1]
)
ec2_instance['Instance Id'] = (raw_ec2_instance.get(
'InstanceId'
))
ec2_instance['Instance Type'] = (raw_ec2_instance.get(
'InstanceType'
))
ec2_instance['Platform'] = (raw_ec2_instance.get(
'Platform'
))
ec2_instance['Private Ip Address'] = (raw_ec2_instance.get(
'PrivateIpAddress'
))
ec2_instance['Public Ip Address'] = (raw_ec2_instance.get(
'PublicIpAddress'
))
ec2_instance['Vpc Id'] = (raw_ec2_instance.get(
'VpcId'
))
ec2_instance['Subnet Id'] = (raw_ec2_instance.get(
'SubnetId'
))
ec2_instances.append(ec2_instance)
return ec2_instances
def get_csv_string_object(data):
stream = io.StringIO()
headers = list(data[0].keys())
writer = csv.DictWriter(stream, fieldnames=headers)
writer.writeheader()
writer.writerows(data)
csv_string_object = stream.getvalue()
return csv_string_object
def put_in_s3(event, csv_string_object):
resource = boto3.resource("s3")
now = datetime.now()
dt_string = now.strftime("%d-%m-%Y-%H-%M-%S")
s3_key = f'organization-ec2-inventory-{dt_string}.csv'
s3_bucket = event.get('s3_bucket', os.environ.get('s3_bucket'))
resource.Object(s3_bucket, s3_key).put(Body=csv_string_object)
def role_arn_to_session(
role_arn, role_session_name, region
):
client = boto3.client("sts")
response = client.assume_role(
RoleArn=role_arn,
RoleSessionName=role_session_name
)
return boto3.Session(
aws_access_key_id=response.get("Credentials").get("AccessKeyId"),
aws_secret_access_key=response.get("Credentials").get("SecretAccessKey"),
aws_session_token=response.get("Credentials").get("SessionToken"),
region_name=region
)
def get_boto3_session(event, account_id, region):
return role_arn_to_session(
role_arn="arn:aws:iam::" + account_id + ":role/" + event.get(
'organization_access_role', 'OrganizationAccountAccessRole'
),
role_session_name=f"{account_id}-crossaccount-role",
region=region
)
def get_ec2_instances(event, account, region_list):
ec2_instances_in_account = []
if account.get('Status') == 'ACTIVE':
for region in region_list:
session = get_boto3_session(event, account.get('Id'), region)
client = session.client('ec2')
raw_ec2_instances = get_ec2_list(client)
ec2_instances_in_region = get_ec2_from_raw(
client, raw_ec2_instances, account.get('Name')
)
ec2_instances_in_account.extend(ec2_instances_in_region)
return ec2_instances_in_account
def lambda_handler(event, context):
client = boto3.client('organizations')
account_list = get_account_list(client)
client = boto3.client('ec2')
region_list = get_region_list(client)
ec2_instances_for_s3 = []
for account in account_list:
ec2_instances = get_ec2_instances(
event, account, region_list
)
ec2_instances_for_s3.extend(ec2_instances)
csv_string_object = get_csv_string_object(ec2_instances_for_s3)
put_in_s3(event, csv_string_object)
Finally, we need an IAM role for this lambda function which will authorize our lambda function on listing account in organization, listing regions and saving objects in S3. We also have to create a S3 bucket. We do not want to work so much. So, we can write a simple Cloudformation template.
AWSTemplateFormatVersion: "2010-09-09"
Parameters:
BucketName:
Description: Name of the bucket
Type: String
Default: 'test-organization-inventory-0987'
Resources:
LambdaRole:
Type: 'AWS::IAM::Role'
Properties:
AssumeRolePolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Principal:
Service:
- lambda.amazonaws.com
Action:
- 'sts:AssumeRole'
Path: /
Policies:
- PolicyName: root
PolicyDocument:
Version: "2012-10-17"
Statement:
- Effect: Allow
Action: 'logs:CreateLogGroup'
Resource: '*'
- Effect: Allow
Action:
- 'logs:CreateLogStream'
- 'logs:PutLogEvents'
Resource: '*'
- Effect: Allow
Action:
- 's3:*'
- 's3-object-lambda:*'
Resource: '*'
- Effect: Allow
Action:
- 'organizations:Describe*'
- 'organizations:List*'
Resource: '*'
- Effect: Allow
Action:
- 'account:GetAlternateContact'
- 'account:GetContactInformation'
- 'account:ListRegions'
Resource: '*'
- Effect: Allow
Action: 'sts:AssumeRole'
Resource: '*'
- Effect: Allow
Action: 'ec2:DescribeRegions'
Resource: '*'
RoleName: 'organization-inventory-role'
InventoryS3Bucket:
Type: AWS::S3::Bucket
Properties:
BucketName: !Ref BucketName
EC2InventoryLambdaFunction:
Type: AWS::Lambda::Function
DependsOn:
- LambdaRole
- InventoryS3Bucket
Properties:
Role: !GetAtt LambdaRole.Arn
Runtime: python3.7
Handler: index.lambda_handler
Timeout: 900
MemorySize: 2048
FunctionName: organization-ec2-inventory
Environment:
Variables:
s3_bucket:
Ref: BucketName
Code:
ZipFile: |
import os
import boto3
import io
import csv
from datetime import datetime
def get_account_list(client):
account_info = client.list_accounts()
account_list = account_info.get('Accounts')
next_token = account_info.get('NextToken')
while next_token != None:
account_info = client.list_accounts(NextToken=next_token)
account_list.extend(account_info.get('Accounts'))
next_token = account_info.get('NextToken')
return account_list
def get_region_list(client):
regions = [region['RegionName']
for region in client.describe_regions()['Regions']]
return regions
def get_instance_list_from_response(response):
instance_list = []
for reservation in response.get('Reservations'):
if 'Instances' in reservation:
instance_list.extend(reservation.get('Instances'))
return instance_list
def get_ec2_list(client):
response = client.describe_instances()
ec2_list = get_instance_list_from_response(response)
next_token = response.get('NextToken')
while next_token != None:
response = client.describe_instances(
NextToken=next_token
)
ec2_list.extend(get_instance_list_from_response(response))
next_token = response.get('NextToken')
return ec2_list
def get_ec2_from_raw(
client, raw_ec2_instances, account_name
):
ec2_instances = []
for raw_ec2_instance in raw_ec2_instances:
ec2_instance = {}
ec2_instance['Account Name'] = account_name
ec2_instance['Region'] = (
raw_ec2_instance.get('Placement').get('AvailabilityZone')[:-1]
)
ec2_instance['Instance Id'] = (raw_ec2_instance.get(
'InstanceId'
))
ec2_instance['Instance Type'] = (raw_ec2_instance.get(
'InstanceType'
))
ec2_instance['Platform'] = (raw_ec2_instance.get(
'Platform'
))
ec2_instance['Private Ip Address'] = (raw_ec2_instance.get(
'PrivateIpAddress'
))
ec2_instance['Public Ip Address'] = (raw_ec2_instance.get(
'PublicIpAddress'
))
ec2_instance['Vpc Id'] = (raw_ec2_instance.get(
'VpcId'
))
ec2_instance['Subnet Id'] = (raw_ec2_instance.get(
'SubnetId'
))
ec2_instances.append(ec2_instance)
return ec2_instances
def get_csv_string_object(data):
stream = io.StringIO()
headers = list(data[0].keys())
writer = csv.DictWriter(stream, fieldnames=headers)
writer.writeheader()
writer.writerows(data)
csv_string_object = stream.getvalue()
return csv_string_object
def put_in_s3(event, csv_string_object):
resource = boto3.resource("s3")
now = datetime.now()
dt_string = now.strftime("%d-%m-%Y-%H-%M-%S")
s3_key = f'organization-ec2-inventory-{dt_string}.csv'
s3_bucket = event.get('s3_bucket', os.environ.get('s3_bucket'))
resource.Object(s3_bucket, s3_key).put(Body=csv_string_object)
def role_arn_to_session(
role_arn, role_session_name, region
):
client = boto3.client("sts")
response = client.assume_role(
RoleArn=role_arn,
RoleSessionName=role_session_name
)
return boto3.Session(
aws_access_key_id=response.get("Credentials").get("AccessKeyId"),
aws_secret_access_key=response.get("Credentials").get("SecretAccessKey"),
aws_session_token=response.get("Credentials").get("SessionToken"),
region_name=region
)
def get_boto3_session(event, account_id, region):
return role_arn_to_session(
role_arn="arn:aws:iam::" + account_id + ":role/" + event.get(
'organization_access_role', 'OrganizationAccountAccessRole'
),
role_session_name=f"{account_id}-crossaccount-role",
region=region
)
def get_ec2_instances(event, account, region_list):
ec2_instances_in_account = []
if account.get('Status') == 'ACTIVE':
for region in region_list:
session = get_boto3_session(event, account.get('Id'), region)
client = session.client('ec2')
raw_ec2_instances = get_ec2_list(client)
ec2_instances_in_region = get_ec2_from_raw(
client, raw_ec2_instances, account.get('Name')
)
ec2_instances_in_account.extend(ec2_instances_in_region)
return ec2_instances_in_account
def lambda_handler(event, context):
client = boto3.client('organizations')
account_list = get_account_list(client)
client = boto3.client('ec2')
region_list = get_region_list(client)
ec2_instances_for_s3 = []
for account in account_list:
ec2_instances = get_ec2_instances(
event, account, region_list
)
ec2_instances_for_s3.extend(ec2_instances)
csv_string_object = get_csv_string_object(ec2_instances_for_s3)
put_in_s3(event, csv_string_object)
Please note that, the bucket name must be unique globally. The permissions in the role can be much better if limit further to maintain east -privilege policy. Now, we can apply the template in cloudformation and our necessary resources will be created. Wooho!! Great job!! We have created the lambda function to build an EC2 inventory and store it in a S3 bucket. We can build a CloudWatch event to trigger the lambda function in a schedule. We can build inventory for any resource by following this procedure. Just follow the boto3 documentation: https://boto3.amazonaws.com/v1/documentation/api/latest/index.html#
Top comments (0)