mirror of
https://github.com/SyncrowIOT/data.git
synced 2025-07-09 22:57:19 +00:00
initial implementation for continuous deployment
This commit is contained in:
31
.github/workflows/deploy-views.yml
vendored
Normal file
31
.github/workflows/deploy-views.yml
vendored
Normal file
@ -0,0 +1,31 @@
|
||||
name: Deploy SQL Views
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
POSTGRES_HOST: ${{ secrets.POSTGRES_HOST }}
|
||||
POSTGRES_PORT: ${{ secrets.POSTGRES_PORT }}
|
||||
POSTGRES_DB: ${{ secrets.POSTGRES_DB }}
|
||||
POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
|
||||
POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
|
||||
|
||||
jobs:
|
||||
deploy-views:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.x'
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install -r requirements.txt
|
||||
|
||||
- name: Deploy SQL views
|
||||
run: python scripts/deploy_views.py
|
31
.gitignore
vendored
Normal file
31
.gitignore
vendored
Normal file
@ -0,0 +1,31 @@
|
||||
# Environment variables
|
||||
.env.*
|
||||
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
.Python
|
||||
env/
|
||||
build/
|
||||
develop-eggs/
|
||||
dist/
|
||||
downloads/
|
||||
eggs/
|
||||
.eggs/
|
||||
lib/
|
||||
lib64/
|
||||
parts/
|
||||
sdist/
|
||||
var/
|
||||
wheels/
|
||||
*.egg-info/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
52
README.md
52
README.md
@ -1,2 +1,54 @@
|
||||
# data
|
||||
All resources involved in data analytics
|
||||
|
||||
## Setup and Usage of deploy_views.py
|
||||
|
||||
### Initial Setup
|
||||
|
||||
1. Create a virtual environment:
|
||||
```bash
|
||||
python3 -m venv venv
|
||||
```
|
||||
|
||||
2. Activate the virtual environment:
|
||||
```bash
|
||||
source venv/bin/activate # On Unix/macOS
|
||||
# or
|
||||
.\venv\Scripts\activate # On Windows
|
||||
```
|
||||
|
||||
3. Install required packages:
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
```
|
||||
|
||||
### Usage
|
||||
|
||||
The `deploy_views.py` script can be used in three ways:
|
||||
|
||||
1. List all SQL files that would be processed (without making changes):
|
||||
```bash
|
||||
python scripts/deploy_views.py --list
|
||||
```
|
||||
|
||||
2. Deploy a specific view:
|
||||
```bash
|
||||
python scripts/deploy_views.py --target view_name
|
||||
# or with .sql extension
|
||||
python scripts/deploy_views.py --target view_name.sql
|
||||
```
|
||||
|
||||
3. Deploy all views:
|
||||
```bash
|
||||
python scripts/deploy_views.py
|
||||
```
|
||||
|
||||
### Important Notes
|
||||
|
||||
- Always ensure the virtual environment is activated before running the script (you'll see `(venv)` in your terminal prompt)
|
||||
- The script requires a `.env` file with the following variables:
|
||||
- POSTGRES_HOST
|
||||
- POSTGRES_PORT
|
||||
- POSTGRES_DB
|
||||
- POSTGRES_USER
|
||||
- POSTGRES_PASSWORD
|
||||
|
2
requirements.txt
Normal file
2
requirements.txt
Normal file
@ -0,0 +1,2 @@
|
||||
psycopg2-binary>=2.9.9
|
||||
python-dotenv>=1.0.0
|
89
scripts/deploy_views.py
Normal file
89
scripts/deploy_views.py
Normal file
@ -0,0 +1,89 @@
|
||||
import os
|
||||
import psycopg2
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from dotenv import load_dotenv
|
||||
|
||||
# Load environment variables from .env file
|
||||
load_dotenv(dotenv_path='.env.azure-development')
|
||||
#load_dotenv(dotenv_path='.env.localhost')
|
||||
|
||||
# Database connection parameters
|
||||
db_params = {
|
||||
'host': os.getenv('POSTGRES_HOST'),
|
||||
'port': os.getenv('POSTGRES_PORT'),
|
||||
'database': os.getenv('POSTGRES_DB'),
|
||||
'user': os.getenv('POSTGRES_USER'),
|
||||
'password': os.getenv('POSTGRES_PASSWORD')
|
||||
}
|
||||
|
||||
def create_or_replace_view(cursor, view_name, sql_content):
|
||||
# First drop the view if it exists
|
||||
drop_sql = f"DROP VIEW IF EXISTS {view_name} CASCADE"
|
||||
cursor.execute(drop_sql)
|
||||
|
||||
# Create view
|
||||
view_sql = f"CREATE VIEW {view_name} AS {sql_content}"
|
||||
cursor.execute(view_sql)
|
||||
|
||||
def main():
|
||||
# Set up argument parser
|
||||
parser = argparse.ArgumentParser(description='Deploy SQL views to database')
|
||||
parser.add_argument('--list', action='store_true', help='Only print the SQL files that would be processed without making changes')
|
||||
parser.add_argument('--target', help='Deploy only a specific SQL file (provide the file name with or without .sql extension)')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Find all .sql files
|
||||
sql_files = list(Path('.').rglob('*.sql'))
|
||||
|
||||
# Filter for specific file if --target is provided
|
||||
if args.target:
|
||||
target = args.target if args.target.endswith('.sql') else f"{args.target}.sql"
|
||||
sql_files = [f for f in sql_files if f.name == target]
|
||||
if not sql_files:
|
||||
print(f"Error: Target file '{target}' not found")
|
||||
exit(1)
|
||||
|
||||
# If list mode, just print the files and exit
|
||||
if args.list:
|
||||
print("SQL files detected:")
|
||||
for sql_file in sql_files:
|
||||
print(f" - {sql_file}")
|
||||
return
|
||||
|
||||
# Connect to the database
|
||||
conn = psycopg2.connect(**db_params)
|
||||
cursor = conn.cursor()
|
||||
|
||||
try:
|
||||
for sql_file in sql_files:
|
||||
# Get view name from file path (remove .sql extension)
|
||||
view_name = sql_file.stem
|
||||
|
||||
# Read SQL content
|
||||
with open(sql_file, 'r') as f:
|
||||
sql_content = f.read().strip()
|
||||
|
||||
# Skip empty files
|
||||
if not sql_content:
|
||||
print(f"Skipping empty file: {sql_file}")
|
||||
continue
|
||||
|
||||
print(f"Creating/replacing view: {view_name}")
|
||||
create_or_replace_view(cursor, view_name, sql_content)
|
||||
|
||||
# Commit all changes
|
||||
conn.commit()
|
||||
print("All views deployed successfully!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {str(e)}")
|
||||
conn.rollback()
|
||||
exit(1)
|
||||
|
||||
finally:
|
||||
cursor.close()
|
||||
conn.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
Reference in New Issue
Block a user