sudo apt update && sudo apt upgrade -y
sudo apt install python3-pip python3-psycopg2 default-libmysqlclient-dev
python3 -m pip install --upgrade pip
pip install --user apache-airflow[amazon] apache-airflow[postgres] apache-airflow[celery] psycopg2-binary apache-airflow-providers-mysql Redis
echo "export AIRFLOW_HOME=${HOME}/airflow" >> .zshrc
airflow db init
airflow.cfg
[core]
executor = CeleryExecutor
load_examples = False
[database]
sql_alchemy_conn = postgresql://myuser:mypassword@my-rds-instance.abc123.us-west-2.rds.amazonaws.com:5432/mydatabase`
[celery]
broker_url = redis://localhost:6379/0
result_backend = redis://localhost:6379/1
Celery Executor: https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/executor/celery.html
airflow db init
airflow users create --username admin --firstname FirstName --lastname LastName --role Admin --email email@example.com
/etc/systemd/system
sudo systemctl enable airflow-webserver.service
sudo systemctl enable airflow-scheduler.service
sudo systemctl enable airflow-celery-flower.service
sudo systemctl enable airflow-celery-worker.service
sudo systemctl start airflow-webserver
sudo systemctl start airflow-scheduler
sudo systemctl start airflow-celery-flower
sudo systemctl start airflow-celery-worker
Top comments (0)