initial implementation for continuous deployment

.github/workflows/deploy-views.yml

@@ -0,0 +1,31 @@
+name: Deploy SQL Views
+
+on:
+  workflow_dispatch:
+
+env:
+  POSTGRES_HOST: ${{ secrets.POSTGRES_HOST }}
+  POSTGRES_PORT: ${{ secrets.POSTGRES_PORT }}
+  POSTGRES_DB: ${{ secrets.POSTGRES_DB }}
+  POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+  POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+
+jobs:
+  deploy-views:
+    runs-on: ubuntu-latest
+    
+    steps:
+    - uses: actions/checkout@v3
+    
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+    
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Deploy SQL views
+      run: python scripts/deploy_views.py

.gitignore

@@ -0,0 +1,31 @@
+# Environment variables
+.env.*
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# IDE
+.idea/
+.vscode/
+*.swp
+*.swo 

README.md

@@ -1,2 +1,54 @@
 # data
 All resources involved in data analytics
+
+## Setup and Usage of deploy_views.py
+
+### Initial Setup
+
+1. Create a virtual environment:
+```bash
+python3 -m venv venv
+```
+
+2. Activate the virtual environment:
+```bash
+source venv/bin/activate  # On Unix/macOS
+# or
+.\venv\Scripts\activate  # On Windows
+```
+
+3. Install required packages:
+```bash
+pip install -r requirements.txt
+```
+
+### Usage
+
+The `deploy_views.py` script can be used in three ways:
+
+1. List all SQL files that would be processed (without making changes):
+```bash
+python scripts/deploy_views.py --list
+```
+
+2. Deploy a specific view:
+```bash
+python scripts/deploy_views.py --target view_name
+# or with .sql extension
+python scripts/deploy_views.py --target view_name.sql
+```
+
+3. Deploy all views:
+```bash
+python scripts/deploy_views.py
+```
+
+### Important Notes
+
+- Always ensure the virtual environment is activated before running the script (you'll see `(venv)` in your terminal prompt)
+- The script requires a `.env` file with the following variables:
+  - POSTGRES_HOST
+  - POSTGRES_PORT
+  - POSTGRES_DB
+  - POSTGRES_USER
+  - POSTGRES_PASSWORD

requirements.txt

@@ -0,0 +1,2 @@
+psycopg2-binary>=2.9.9
+python-dotenv>=1.0.0 

scripts/deploy_views.py

@@ -0,0 +1,89 @@
+import os
+import psycopg2
+import argparse
+from pathlib import Path
+from dotenv import load_dotenv
+
+# Load environment variables from .env file
+load_dotenv(dotenv_path='.env.azure-development')
+#load_dotenv(dotenv_path='.env.localhost')
+
+# Database connection parameters
+db_params = {
+    'host': os.getenv('POSTGRES_HOST'),
+    'port': os.getenv('POSTGRES_PORT'),
+    'database': os.getenv('POSTGRES_DB'),
+    'user': os.getenv('POSTGRES_USER'),
+    'password': os.getenv('POSTGRES_PASSWORD')
+}
+
+def create_or_replace_view(cursor, view_name, sql_content):
+    # First drop the view if it exists
+    drop_sql = f"DROP VIEW IF EXISTS {view_name} CASCADE"
+    cursor.execute(drop_sql)
+    
+    # Create view
+    view_sql = f"CREATE VIEW {view_name} AS {sql_content}"
+    cursor.execute(view_sql)
+
+def main():
+    # Set up argument parser
+    parser = argparse.ArgumentParser(description='Deploy SQL views to database')
+    parser.add_argument('--list', action='store_true', help='Only print the SQL files that would be processed without making changes')
+    parser.add_argument('--target', help='Deploy only a specific SQL file (provide the file name with or without .sql extension)')
+    args = parser.parse_args()
+
+    # Find all .sql files
+    sql_files = list(Path('.').rglob('*.sql'))
+    
+    # Filter for specific file if --target is provided
+    if args.target:
+        target = args.target if args.target.endswith('.sql') else f"{args.target}.sql"
+        sql_files = [f for f in sql_files if f.name == target]
+        if not sql_files:
+            print(f"Error: Target file '{target}' not found")
+            exit(1)
+
+    # If list mode, just print the files and exit
+    if args.list:
+        print("SQL files detected:")
+        for sql_file in sql_files:
+            print(f"  - {sql_file}")
+        return
+
+    # Connect to the database
+    conn = psycopg2.connect(**db_params)
+    cursor = conn.cursor()
+    
+    try:
+        for sql_file in sql_files:
+            # Get view name from file path (remove .sql extension)
+            view_name = sql_file.stem
+            
+            # Read SQL content
+            with open(sql_file, 'r') as f:
+                sql_content = f.read().strip()
+            
+            # Skip empty files
+            if not sql_content:
+                print(f"Skipping empty file: {sql_file}")
+                continue
+                
+            print(f"Creating/replacing view: {view_name}")
+            create_or_replace_view(cursor, view_name, sql_content)
+        
+        # Commit all changes
+        conn.commit()
+        print("All views deployed successfully!")
+        
+    except Exception as e:
+        print(f"Error: {str(e)}")
+        conn.rollback()
+        exit(1)
+        
+    finally:
+        cursor.close()
+        conn.close()
+
+if __name__ == "__main__":
+    main()