From 691f9972f2c771fcb7c9ff0b2d037a350beabedd Mon Sep 17 00:00:00 2001 From: alex Date: Thu, 6 Feb 2025 12:54:27 +0100 Subject: [PATCH] initial implementation for continuous deployment --- .github/workflows/deploy-views.yml | 31 +++++++++++ .gitignore | 31 +++++++++++ README.md | 52 +++++++++++++++++ requirements.txt | 2 + scripts/deploy_views.py | 89 ++++++++++++++++++++++++++++++ 5 files changed, 205 insertions(+) create mode 100644 .github/workflows/deploy-views.yml create mode 100644 .gitignore create mode 100644 requirements.txt create mode 100644 scripts/deploy_views.py diff --git a/.github/workflows/deploy-views.yml b/.github/workflows/deploy-views.yml new file mode 100644 index 0000000..08edc11 --- /dev/null +++ b/.github/workflows/deploy-views.yml @@ -0,0 +1,31 @@ +name: Deploy SQL Views + +on: + workflow_dispatch: + +env: + POSTGRES_HOST: ${{ secrets.POSTGRES_HOST }} + POSTGRES_PORT: ${{ secrets.POSTGRES_PORT }} + POSTGRES_DB: ${{ secrets.POSTGRES_DB }} + POSTGRES_USER: ${{ secrets.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }} + +jobs: + deploy-views: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Deploy SQL views + run: python scripts/deploy_views.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d534343 --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +# Environment variables +.env.* + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# IDE +.idea/ +.vscode/ +*.swp +*.swo \ No newline at end of file diff --git a/README.md b/README.md index 8ca998c..1178cdf 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,54 @@ # data All resources involved in data analytics + +## Setup and Usage of deploy_views.py + +### Initial Setup + +1. Create a virtual environment: +```bash +python3 -m venv venv +``` + +2. Activate the virtual environment: +```bash +source venv/bin/activate # On Unix/macOS +# or +.\venv\Scripts\activate # On Windows +``` + +3. Install required packages: +```bash +pip install -r requirements.txt +``` + +### Usage + +The `deploy_views.py` script can be used in three ways: + +1. List all SQL files that would be processed (without making changes): +```bash +python scripts/deploy_views.py --list +``` + +2. Deploy a specific view: +```bash +python scripts/deploy_views.py --target view_name +# or with .sql extension +python scripts/deploy_views.py --target view_name.sql +``` + +3. Deploy all views: +```bash +python scripts/deploy_views.py +``` + +### Important Notes + +- Always ensure the virtual environment is activated before running the script (you'll see `(venv)` in your terminal prompt) +- The script requires a `.env` file with the following variables: + - POSTGRES_HOST + - POSTGRES_PORT + - POSTGRES_DB + - POSTGRES_USER + - POSTGRES_PASSWORD diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..df39648 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +psycopg2-binary>=2.9.9 +python-dotenv>=1.0.0 \ No newline at end of file diff --git a/scripts/deploy_views.py b/scripts/deploy_views.py new file mode 100644 index 0000000..091027d --- /dev/null +++ b/scripts/deploy_views.py @@ -0,0 +1,89 @@ +import os +import psycopg2 +import argparse +from pathlib import Path +from dotenv import load_dotenv + +# Load environment variables from .env file +load_dotenv(dotenv_path='.env.azure-development') +#load_dotenv(dotenv_path='.env.localhost') + +# Database connection parameters +db_params = { + 'host': os.getenv('POSTGRES_HOST'), + 'port': os.getenv('POSTGRES_PORT'), + 'database': os.getenv('POSTGRES_DB'), + 'user': os.getenv('POSTGRES_USER'), + 'password': os.getenv('POSTGRES_PASSWORD') +} + +def create_or_replace_view(cursor, view_name, sql_content): + # First drop the view if it exists + drop_sql = f"DROP VIEW IF EXISTS {view_name} CASCADE" + cursor.execute(drop_sql) + + # Create view + view_sql = f"CREATE VIEW {view_name} AS {sql_content}" + cursor.execute(view_sql) + +def main(): + # Set up argument parser + parser = argparse.ArgumentParser(description='Deploy SQL views to database') + parser.add_argument('--list', action='store_true', help='Only print the SQL files that would be processed without making changes') + parser.add_argument('--target', help='Deploy only a specific SQL file (provide the file name with or without .sql extension)') + args = parser.parse_args() + + # Find all .sql files + sql_files = list(Path('.').rglob('*.sql')) + + # Filter for specific file if --target is provided + if args.target: + target = args.target if args.target.endswith('.sql') else f"{args.target}.sql" + sql_files = [f for f in sql_files if f.name == target] + if not sql_files: + print(f"Error: Target file '{target}' not found") + exit(1) + + # If list mode, just print the files and exit + if args.list: + print("SQL files detected:") + for sql_file in sql_files: + print(f" - {sql_file}") + return + + # Connect to the database + conn = psycopg2.connect(**db_params) + cursor = conn.cursor() + + try: + for sql_file in sql_files: + # Get view name from file path (remove .sql extension) + view_name = sql_file.stem + + # Read SQL content + with open(sql_file, 'r') as f: + sql_content = f.read().strip() + + # Skip empty files + if not sql_content: + print(f"Skipping empty file: {sql_file}") + continue + + print(f"Creating/replacing view: {view_name}") + create_or_replace_view(cursor, view_name, sql_content) + + # Commit all changes + conn.commit() + print("All views deployed successfully!") + + except Exception as e: + print(f"Error: {str(e)}") + conn.rollback() + exit(1) + + finally: + cursor.close() + conn.close() + +if __name__ == "__main__": + main() \ No newline at end of file