Updated README.md file with project details #17
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI/CD - AWS Glue Delta Lake Project | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| force_deploy: | |
| description: "Deploy Glue scripts even if validation fails" | |
| required: true | |
| default: "false" | |
| type: choice | |
| options: | |
| - "false" | |
| - "true" | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| AWS_REGION: us-east-1 | |
| GLUE_SCRIPT_BUCKET: wistia-data-bucket-001 | |
| GLUE_SCRIPT_PREFIX: scripts/glue_jobs | |
| jobs: | |
| validate-python: | |
| name: Validate Python code | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install development dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-dev.txt | |
| - name: Run Ruff checks | |
| run: | | |
| ruff check glue_jobs streamlit_app tests | |
| - name: Verify dashboard imports | |
| run: | | |
| python -c "import altair, deltalake, pandas, streamlit" | |
| - name: Run unit tests | |
| run: | | |
| python -m unittest discover -s tests -v | |
| deploy-glue-scripts: | |
| name: Deploy Glue scripts to S3 | |
| runs-on: ubuntu-latest | |
| needs: validate-python | |
| if: | | |
| always() && | |
| ( | |
| (github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.validate-python.result == 'success') || | |
| (github.event_name == 'workflow_dispatch' && inputs.force_deploy == 'true') | |
| ) | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::778277577883:role/wistia-github-actions-glue-deploy-role | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Upload Glue job scripts to S3 | |
| run: | | |
| aws s3 sync glue_jobs/ s3://${GLUE_SCRIPT_BUCKET}/${GLUE_SCRIPT_PREFIX}/ \ | |
| --delete \ | |
| --exclude "__pycache__/*" \ | |
| --exclude "*.pyc" |