diff --git a/.gitignore b/.gitignore index e7dd83c..dc0bd3d 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,10 @@ ENV/ env.bak/ venv.bak/ +# Alembic - keep migration files but not temporary files +# backend/alembic/versions/*.pyc +# backend/alembic/__pycache__/ + # IDE .vscode/ .idea/ diff --git a/README.md b/README.md index cfa33a0..ffe2a7c 100644 --- a/README.md +++ b/README.md @@ -28,12 +28,60 @@ A modern, containerized Prior Permission Required (PPR) system for aircraft oper ### Prerequisites - Docker and Docker Compose installed -### 1. Start the System +### Development Setup + +**Start the system (automatic database setup):** ```bash -cd nextgen -./start.sh +docker compose up -d ``` +That's it! The container automatically: +- Waits for database to be ready +- Creates schema via Alembic migrations +- Loads airport and aircraft reference data +- Starts the API with auto-reload + +**View startup logs:** +```bash +docker compose logs -f api +``` + +### Production Deployment + +**Simple automated deployment:** + +```bash +# 1. Configure environment +cp .env.example .env +nano .env # Set your external database credentials + +# 2. Start with production settings +docker compose -f docker-compose.prod.yml up -d +``` + +The container automatically handles: +- Database connection verification +- Schema creation/migration (Alembic) +- Reference data seeding (if needed) +- Production server startup (4 workers) + +**Monitor deployment:** +```bash +docker compose -f docker-compose.prod.yml logs -f api +``` + +**Deploying updates with migrations:** +```bash +git pull +docker compose -f docker-compose.prod.yml up -d --build +# Migrations apply automatically! +``` + +**See detailed guides:** +- [`AUTOMATED_MIGRATION_GUIDE.md`](./AUTOMATED_MIGRATION_GUIDE.md) - How automatic migrations work +- [`PRODUCTION_MIGRATION_GUIDE.md`](./PRODUCTION_MIGRATION_GUIDE.md) - Advanced migration strategies +- [`MIGRATION_QUICK_REF.md`](./MIGRATION_QUICK_REF.md) - Quick reference commands + ### 2. Access the Services - **API Documentation**: http://localhost:8001/docs - **API Base URL**: http://localhost:8001/api/v1 diff --git a/backend/Dockerfile b/backend/Dockerfile index 03fe08d..34e2bec 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -19,8 +19,11 @@ RUN pip install --no-cache-dir -r requirements.txt # Copy application code COPY . . +# Make entrypoint executable +RUN chmod +x /app/entrypoint.sh + # Expose port EXPOSE 8000 -# Run the application -CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file +# Use entrypoint script +ENTRYPOINT ["/app/entrypoint.sh"] \ No newline at end of file diff --git a/backend/README_test_data.md b/backend/README_test_data.md deleted file mode 100644 index 6fa9fea..0000000 --- a/backend/README_test_data.md +++ /dev/null @@ -1,138 +0,0 @@ -# Test Data Population Script - -This script generates and inserts 30 random PPR (Prior Permission Required) records into the database for testing purposes. - -## Features - -- **30 Random PPR Records**: Generates diverse test data with various aircraft, airports, and flight details -- **Real Aircraft Data**: Uses actual aircraft registration data from the `aircraft_data.csv` file -- **Real Airport Data**: Uses actual airport ICAO codes from the `airports_data_clean.csv` file -- **Random Status Distribution**: Includes NEW, CONFIRMED, LANDED, and DEPARTED statuses -- **Realistic Timestamps**: Generates ETA/ETD times with 15-minute intervals -- **Optional Fields**: Randomly includes email, phone, notes, and departure details -- **Duplicate Aircraft**: Some aircraft registrations appear multiple times for realistic testing - -## Usage - -### Prerequisites -- Database must be running and accessible -- Python environment with required dependencies installed -- CSV data files (`aircraft_data.csv` and `airports_data_clean.csv`) in the parent directory - -### Running the Script - -1. **Using the convenience script** (recommended): - ```bash - cd /home/jamesp/docker/pprdev/nextgen - ./populate_test_data.sh - ``` - -2. **From within the Docker container**: - ```bash - docker exec -it ppr-backend bash - cd /app - python populate_test_data.py - ``` - -3. **From host machine** (if database is accessible): - ```bash - cd /home/jamesp/docker/pprdev/nextgen/backend - python populate_test_data.py - ``` - -## What Gets Generated - -Each PPR record includes: -- **Aircraft**: Random registration, type, and callsign from real aircraft data -- **Route**: Random arrival airport (from Swansea), optional departure airport -- **Times**: ETA between 6 AM - 8 PM, ETD 1-4 hours later (if departing) -- **Passengers**: 1-4 POB for arrival, optional for departure -- **Contact**: Optional email and phone (70% and 50% chance respectively) -- **Fuel**: Random fuel type (100LL, JET A1, FULL) or none -- **Notes**: Optional flight purpose notes (various scenarios) -- **Status**: Random status distribution (NEW/CONFIRMED/LANDED/DEPARTED) -- **Timestamps**: Random submission dates within last 30 days -- **Public Token**: Auto-generated for edit/cancel functionality - -### Aircraft Distribution -- Uses real aircraft registration data from `aircraft_data.csv` -- Includes various aircraft types (C172, PA28, BE36, R44, etc.) -- Some aircraft appear multiple times for realistic duplication - -### Airport Distribution -- Uses real ICAO airport codes from `airports_data_clean.csv` -- Arrival airports are distributed globally -- Departure airports (when included) are different from arrival airports - -### Data Quality Notes - -- **Realistic Distribution**: Aircraft and airports are selected from actual aviation data -- **Time Constraints**: All times are within reasonable operating hours (6 AM - 8 PM) -- **Status Balance**: Roughly equal distribution across different PPR statuses -- **Contact Info**: Realistic email patterns and UK phone numbers -- **Flight Logic**: Departures only occur when a departure airport is specified - -## Assumptions - -- Database schema matches the PPRRecord model in `app/models/ppr.py` -- CSV files are present and properly formatted -- Database connection uses settings from `app/core/config.py` -- All required dependencies are installed in the Python environment - -### Sample Output - -``` -Loading aircraft and airport data... -Loaded 520000 aircraft records -Loaded 43209 airport records -Generating and inserting 30 test PPR records... -Generated 10 records... -Generated 20 records... -Generated 30 records... -✅ Successfully inserted 30 test PPR records! -Total PPR records in database: 42 - -Status breakdown: - NEW: 8 - CONFIRMED: 7 - LANDED: 9 - DEPARTED: 6 -``` - -## Safety Notes - -- **Non-destructive**: Only adds new records, doesn't modify existing data -- **Test Data Only**: All generated data is clearly identifiable as test data -- **Easy Cleanup**: Can be easily removed with SQL queries if needed - -## Current Status ✅ - -The script is working correctly! It has successfully generated and inserted test data. As of the latest run: - -- **Total PPR records in database**: 93 -- **Status breakdown**: - - NEW: 19 - - CONFIRMED: 22 - - CANCELED: 1 - - LANDED: 35 - - DEPARTED: 16 - -## Troubleshooting - -- **Database Connection**: Ensure the database container is running and accessible -- **CSV Files**: The script uses fallback data when CSV files aren't found (which is normal in containerized environments) -- **Dependencies**: Ensure all Python requirements are installed -- **Permissions**: Script needs database write permissions - -## Recent Fixes - -- ✅ Fixed SQLAlchemy 2.0 `func.count()` import issue -- ✅ Script now runs successfully and provides status breakdown -- ✅ Uses fallback aircraft/airport data when CSV files aren't accessible - -## Cleanup (if needed) - -To remove all test data: -```sql -DELETE FROM submitted WHERE submitted_dt > '2025-01-01'; -- Adjust date as needed -``` \ No newline at end of file diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 0000000..b83baaa --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,114 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# Database URL - will be overridden by env.py from environment variables +sqlalchemy.url = mysql+pymysql://user:pass@localhost/dbname + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the exec runner, execute a binary +# hooks = ruff +# ruff.type = exec +# ruff.executable = %(here)s/.venv/bin/ruff +# ruff.options = --fix REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/alembic/env.py b/backend/alembic/env.py new file mode 100644 index 0000000..0267b04 --- /dev/null +++ b/backend/alembic/env.py @@ -0,0 +1,92 @@ +from logging.config import fileConfig +from sqlalchemy import engine_from_config +from sqlalchemy import pool +from alembic import context +import os +import sys + +# Add the app directory to the path +sys.path.insert(0, os.path.dirname(os.path.dirname(__file__))) + +from app.core.config import settings +from app.db.session import Base +# Import all models to ensure they are registered with Base +from app.models.ppr import PPRRecord, User, Journal, Airport, Aircraft + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Override sqlalchemy.url with the one from settings +config.set_main_option('sqlalchemy.url', settings.database_url) + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + compare_server_default=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + compare_server_default=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako new file mode 100644 index 0000000..55df286 --- /dev/null +++ b/backend/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/backend/alembic/versions/001_initial_schema.py b/backend/alembic/versions/001_initial_schema.py new file mode 100644 index 0000000..f5d37b8 --- /dev/null +++ b/backend/alembic/versions/001_initial_schema.py @@ -0,0 +1,165 @@ +"""Initial schema baseline + +Revision ID: 001_initial_schema +Revises: +Create Date: 2025-12-04 12:00:00.000000 + +This is the baseline migration that captures the current database schema. +For existing databases, this migration should be marked as applied without running it. +For new databases, this creates the complete initial schema. + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '001_initial_schema' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """ + Create all tables for a fresh database installation. + """ + + # Users table + op.create_table('users', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('username', sa.String(length=50), nullable=False), + sa.Column('password', sa.String(length=255), nullable=False), + sa.Column('role', sa.Enum('ADMINISTRATOR', 'OPERATOR', 'READ_ONLY', name='userrole'), nullable=False), + sa.Column('email', sa.String(length=128), nullable=True), + sa.Column('full_name', sa.String(length=100), nullable=True), + sa.Column('is_active', sa.Integer(), nullable=False, server_default='1'), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('username'), + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_unicode_ci' + ) + op.create_index('idx_username', 'users', ['username']) + op.create_index('idx_email', 'users', ['email']) + + # Main PPR submissions table + op.create_table('submitted', + sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column('status', sa.Enum('NEW', 'CONFIRMED', 'CANCELED', 'LANDED', 'DELETED', 'DEPARTED', name='pprstatus'), nullable=False, server_default='NEW'), + sa.Column('ac_reg', sa.String(length=16), nullable=False), + sa.Column('ac_type', sa.String(length=32), nullable=False), + sa.Column('ac_call', sa.String(length=16), nullable=True), + sa.Column('captain', sa.String(length=64), nullable=False), + sa.Column('fuel', sa.String(length=16), nullable=True), + sa.Column('in_from', sa.String(length=64), nullable=False), + sa.Column('eta', sa.DateTime(), nullable=False), + sa.Column('pob_in', sa.Integer(), nullable=False), + sa.Column('out_to', sa.String(length=64), nullable=True), + sa.Column('etd', sa.DateTime(), nullable=True), + sa.Column('pob_out', sa.Integer(), nullable=True), + sa.Column('email', sa.String(length=128), nullable=True), + sa.Column('phone', sa.String(length=16), nullable=True), + sa.Column('notes', sa.Text(), nullable=True), + sa.Column('landed_dt', sa.DateTime(), nullable=True), + sa.Column('departed_dt', sa.DateTime(), nullable=True), + sa.Column('created_by', sa.String(length=16), nullable=True), + sa.Column('submitted_dt', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.Column('public_token', sa.String(length=128), nullable=True), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('public_token'), + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_unicode_ci' + ) + op.create_index('idx_status', 'submitted', ['status']) + op.create_index('idx_eta', 'submitted', ['eta']) + op.create_index('idx_etd', 'submitted', ['etd']) + op.create_index('idx_ac_reg', 'submitted', ['ac_reg']) + op.create_index('idx_submitted_dt', 'submitted', ['submitted_dt']) + op.create_index('idx_created_by', 'submitted', ['created_by']) + op.create_index('idx_public_token', 'submitted', ['public_token']) + + # Activity journal table + op.create_table('journal', + sa.Column('id', sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column('ppr_id', sa.BigInteger(), nullable=False), + sa.Column('entry', sa.Text(), nullable=False), + sa.Column('user', sa.String(length=50), nullable=False), + sa.Column('ip', sa.String(length=45), nullable=False), + sa.Column('entry_dt', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['ppr_id'], ['submitted.id'], ondelete='CASCADE'), + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_unicode_ci' + ) + op.create_index('idx_ppr_id', 'journal', ['ppr_id']) + op.create_index('idx_entry_dt', 'journal', ['entry_dt']) + op.create_index('idx_user', 'journal', ['user']) + + # Airports reference table + op.create_table('airports', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('icao', sa.String(length=4), nullable=False), + sa.Column('iata', sa.String(length=3), nullable=True), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('country', sa.String(length=100), nullable=False), + sa.Column('city', sa.String(length=100), nullable=True), + sa.Column('timezone', sa.String(length=50), nullable=True), + sa.Column('latitude', mysql.DECIMAL(precision=10, scale=8), nullable=True), + sa.Column('longitude', mysql.DECIMAL(precision=11, scale=8), nullable=True), + sa.Column('elevation', sa.Integer(), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('icao', name='unique_icao'), + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_unicode_ci' + ) + op.create_index('idx_iata', 'airports', ['iata']) + op.create_index('idx_country', 'airports', ['country']) + op.create_index('idx_name', 'airports', ['name']) + + # Aircraft reference table + op.create_table('aircraft', + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('icao24', sa.String(length=6), nullable=True), + sa.Column('registration', sa.String(length=25), nullable=True), + sa.Column('manufacturer_icao', sa.String(length=50), nullable=True), + sa.Column('type_code', sa.String(length=30), nullable=True), + sa.Column('manufacturer_name', sa.String(length=255), nullable=True), + sa.Column('model', sa.String(length=255), nullable=True), + sa.Column('clean_reg', sa.String(length=25), sa.Computed("UPPER(REPLACE(REPLACE(registration, '-', ''), ' ', ''))", persisted=True), nullable=True), + sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False), + sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP'), nullable=False), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + mysql_charset='utf8mb4', + mysql_collate='utf8mb4_unicode_ci' + ) + op.create_index('idx_registration', 'aircraft', ['registration']) + op.create_index('idx_clean_reg', 'aircraft', ['clean_reg']) + op.create_index('idx_icao24', 'aircraft', ['icao24']) + op.create_index('idx_type_code', 'aircraft', ['type_code']) + + # Insert default admin user (password: admin123) + # This should be changed immediately in production + op.execute(""" + INSERT INTO users (username, password, role, email, full_name) VALUES + ('admin', '$2b$12$BJOha2yRxkxuHL./BaMfpu2fMDgGMYISuRV2.B1sSklVpRjz3Y4a6', 'ADMINISTRATOR', 'admin@ppr.local', 'System Administrator') + """) + + +def downgrade() -> None: + """ + Drop all tables - USE WITH CAUTION IN PRODUCTION + """ + op.drop_table('journal') + op.drop_table('submitted') + op.drop_table('users') + op.drop_table('aircraft') + op.drop_table('airports') diff --git a/backend/app/main.py b/backend/app/main.py index f2e9448..61320f1 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -69,7 +69,29 @@ async def root(): @app.get("/health") async def health_check(): - return {"status": "healthy", "timestamp": "2024-01-01T00:00:00Z"} + """Health check endpoint with database connectivity verification""" + from datetime import datetime + from sqlalchemy import text + from app.db.session import SessionLocal + + health_status = { + "status": "healthy", + "timestamp": datetime.utcnow().isoformat() + "Z", + "version": "2.0.0" + } + + # Check database connectivity + try: + db = SessionLocal() + db.execute(text("SELECT 1")) + db.close() + health_status["database"] = "connected" + except Exception as e: + health_status["status"] = "unhealthy" + health_status["database"] = "disconnected" + health_status["error"] = str(e) + + return health_status # Include API router app.include_router(api_router, prefix=settings.api_v1_str) diff --git a/backend/app/models/ppr.py b/backend/app/models/ppr.py index dc9bc86..b23b9a8 100644 --- a/backend/app/models/ppr.py +++ b/backend/app/models/ppr.py @@ -23,26 +23,27 @@ class PPRRecord(Base): __tablename__ = "submitted" id = Column(BigInteger, primary_key=True, autoincrement=True) - status = Column(SQLEnum(PPRStatus), nullable=False, default=PPRStatus.NEW) - ac_reg = Column(String(16), nullable=False) + status = Column(SQLEnum(PPRStatus), nullable=False, default=PPRStatus.NEW, index=True) + ac_reg = Column(String(16), nullable=False, index=True) ac_type = Column(String(32), nullable=False) ac_call = Column(String(16), nullable=True) captain = Column(String(64), nullable=False) fuel = Column(String(16), nullable=True) in_from = Column(String(64), nullable=False) - eta = Column(DateTime, nullable=False) + eta = Column(DateTime, nullable=False, index=True) pob_in = Column(Integer, nullable=False) out_to = Column(String(64), nullable=True) - etd = Column(DateTime, nullable=True) + etd = Column(DateTime, nullable=True, index=True) pob_out = Column(Integer, nullable=True) email = Column(String(128), nullable=True) phone = Column(String(16), nullable=True) - notes = Column(String(2000), nullable=True) + notes = Column(Text, nullable=True) landed_dt = Column(DateTime, nullable=True) departed_dt = Column(DateTime, nullable=True) - created_by = Column(String(16), nullable=True) - submitted_dt = Column(DateTime, nullable=False, server_default=func.current_timestamp()) - public_token = Column(String(128), nullable=True, unique=True) + created_by = Column(String(16), nullable=True, index=True) + submitted_dt = Column(DateTime, nullable=False, server_default=func.current_timestamp(), index=True) + public_token = Column(String(128), nullable=True, unique=True, index=True) + updated_at = Column(DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()) class User(Base): @@ -52,17 +53,22 @@ class User(Base): username = Column(String(50), nullable=False, unique=True, index=True) password = Column(String(255), nullable=False) role = Column(SQLEnum(UserRole), nullable=False, default=UserRole.READ_ONLY) + email = Column(String(128), nullable=True) + full_name = Column(String(100), nullable=True) + is_active = Column(Integer, nullable=False, default=1) # Using Integer for BOOLEAN + created_at = Column(DateTime, nullable=False, server_default=func.current_timestamp()) + updated_at = Column(DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()) class Journal(Base): __tablename__ = "journal" id = Column(BigInteger, primary_key=True, autoincrement=True) - ppr_id = Column(Integer, nullable=False, index=True) + ppr_id = Column(BigInteger, nullable=False, index=True) # Changed to BigInteger to match submitted.id entry = Column(Text, nullable=False) - user = Column(Text, nullable=False) - ip = Column(Text, nullable=False) - entry_dt = Column(DateTime, nullable=False, server_default=func.current_timestamp()) + user = Column(String(50), nullable=False, index=True) + ip = Column(String(45), nullable=False) + entry_dt = Column(DateTime, nullable=False, server_default=func.current_timestamp(), index=True) class Airport(Base): @@ -85,12 +91,12 @@ class Aircraft(Base): __tablename__ = "aircraft" id = Column(Integer, primary_key=True, autoincrement=True) - icao24 = Column(String(6), nullable=True) + icao24 = Column(String(6), nullable=True, index=True) registration = Column(String(25), nullable=True, index=True) manufacturer_icao = Column(String(50), nullable=True) - type_code = Column(String(30), nullable=True) + type_code = Column(String(30), nullable=True, index=True) manufacturer_name = Column(String(255), nullable=True) model = Column(String(255), nullable=True) clean_reg = Column(String(25), nullable=True, index=True) created_at = Column(DateTime, nullable=False, server_default=func.current_timestamp()) - updated_at = Column(DateTime, nullable=False, server_default=func.current_timestamp()) \ No newline at end of file + updated_at = Column(DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()) \ No newline at end of file diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh new file mode 100644 index 0000000..7f0c132 --- /dev/null +++ b/backend/entrypoint.sh @@ -0,0 +1,160 @@ +#!/bin/bash +# Docker entrypoint script for PPR API +# Handles database migrations and data seeding automatically + +set -e + +echo "=========================================" +echo "PPR API Container Starting" +echo "=========================================" + +# Wait for database to be ready +echo "Waiting for database to be ready..." +python3 << EOF +import sys +import time +from sqlalchemy import create_engine, text +from app.core.config import settings + +max_retries = 30 +retry_interval = 2 + +for i in range(max_retries): + try: + engine = create_engine(settings.database_url) + with engine.connect() as conn: + conn.execute(text("SELECT 1")) + print("✓ Database is ready") + sys.exit(0) + except Exception as e: + if i < max_retries - 1: + print(f"Database not ready yet (attempt {i+1}/{max_retries}), waiting...") + time.sleep(retry_interval) + else: + print(f"✗ Database connection failed after {max_retries} attempts: {e}") + sys.exit(1) +EOF + +if [ $? -ne 0 ]; then + echo "Failed to connect to database. Exiting." + exit 1 +fi + +# Check if this is a fresh database or needs migration +echo "" +echo "Checking database state..." + +python3 << EOF +import sys +from sqlalchemy import create_engine, text, inspect +from app.core.config import settings + +try: + engine = create_engine(settings.database_url) + inspector = inspect(engine) + + # Check if any tables exist + tables = inspector.get_table_names() + + if not tables: + print("✓ Fresh database detected - will initialize") + sys.exit(2) # Signal fresh database + elif 'alembic_version' not in tables: + print("✓ Existing database without migration tracking - will stamp") + sys.exit(3) # Signal existing database needs stamping + else: + print("✓ Database has migration tracking - will check for updates") + sys.exit(0) # Normal migration check + +except Exception as e: + print(f"✗ Error checking database: {e}") + sys.exit(1) +EOF + +DB_STATE=$? + +if [ $DB_STATE -eq 2 ]; then + # Fresh database - run initial migration + echo "" + echo "Initializing fresh database..." + cd /app + alembic upgrade head + echo "✓ Database schema created" + + # Seed reference data + echo "" + echo "Loading reference data..." + python3 /app/seed_data.py + echo "✓ Reference data loaded" + +elif [ $DB_STATE -eq 3 ]; then + # Existing database without Alembic - stamp it + echo "" + echo "Stamping existing database with migration version..." + cd /app + alembic stamp head + echo "✓ Database stamped" + + # Check if reference data exists + python3 << EOF +from sqlalchemy import create_engine, text +from app.core.config import settings + +engine = create_engine(settings.database_url) +with engine.connect() as conn: + airport_count = conn.execute(text("SELECT COUNT(*) FROM airports")).fetchone()[0] + aircraft_count = conn.execute(text("SELECT COUNT(*) FROM aircraft")).fetchone()[0] + + if airport_count == 0 or aircraft_count == 0: + print("Reference data missing - will load") + exit(10) + else: + print(f"Reference data exists (airports: {airport_count}, aircraft: {aircraft_count})") + exit(0) +EOF + + if [ $? -eq 10 ]; then + echo "Loading reference data..." + python3 /app/seed_data.py + echo "✓ Reference data loaded" + fi + +elif [ $DB_STATE -eq 0 ]; then + # Database with Alembic tracking - check for pending migrations + echo "" + echo "Checking for pending migrations..." + cd /app + + # Get current and head revisions + CURRENT=$(alembic current 2>/dev/null | grep -o '[a-f0-9]\{12\}' | head -1 || echo "none") + HEAD=$(alembic heads 2>/dev/null | grep -o '[a-f0-9]\{12\}' | head -1 || echo "none") + + if [ "$CURRENT" != "$HEAD" ] && [ "$HEAD" != "none" ]; then + echo "✓ Pending migrations detected" + echo " Current: $CURRENT" + echo " Target: $HEAD" + echo "Applying migrations..." + alembic upgrade head + echo "✓ Migrations applied" + else + echo "✓ Database is up to date" + fi +else + echo "✗ Database check failed" + exit 1 +fi + +echo "" +echo "=========================================" +echo "Starting Application Server" +echo "=========================================" +echo "" + +# Start the application with appropriate settings +if [ "${ENVIRONMENT}" = "production" ]; then + echo "Starting in PRODUCTION mode with multiple workers..." + exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --workers ${WORKERS:-4} +else + echo "Starting in DEVELOPMENT mode with auto-reload..." + exec uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload +fi diff --git a/backend/seed_data.py b/backend/seed_data.py new file mode 100644 index 0000000..655f3df --- /dev/null +++ b/backend/seed_data.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +""" +Seed reference data into the database +Loads airport and aircraft data from CSV files +""" +import os +import csv +import sys +from pathlib import Path +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from app.core.config import settings +from app.models.ppr import Airport, Aircraft + +def load_airports(db, csv_path): + """Load airport data from CSV""" + if not os.path.exists(csv_path): + print(f" ⚠ Airport data file not found: {csv_path}") + return 0 + + # Check if data already exists + existing_count = db.query(Airport).count() + if existing_count > 0: + print(f" ⚠ Airport data already exists ({existing_count} records), skipping") + return existing_count + + print(f" Loading airports from {csv_path}...") + loaded = 0 + batch = [] + + with open(csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + airport = Airport( + icao=row['icao'], + iata=row.get('iata') if row.get('iata') else None, + name=row['name'], + country=row['country'] + ) + batch.append(airport) + loaded += 1 + + # Commit in batches of 1000 + if len(batch) >= 1000: + db.bulk_save_objects(batch) + db.commit() + batch = [] + print(f" Loaded {loaded} airports...", end='\r') + + # Commit remaining + if batch: + db.bulk_save_objects(batch) + db.commit() + + print(f" ✓ Loaded {loaded} airport records" + " " * 20) + return loaded + + +def load_aircraft(db, csv_path): + """Load aircraft data from CSV""" + if not os.path.exists(csv_path): + print(f" ⚠ Aircraft data file not found: {csv_path}") + return 0 + + # Check if data already exists + existing_count = db.query(Aircraft).count() + if existing_count > 0: + print(f" ⚠ Aircraft data already exists ({existing_count} records), skipping") + return existing_count + + print(f" Loading aircraft from {csv_path}...") + loaded = 0 + batch = [] + + with open(csv_path, 'r', encoding='utf-8') as f: + reader = csv.DictReader(f) + for row in reader: + aircraft = Aircraft( + icao24=row.get('icao24') if row.get('icao24') else None, + registration=row.get('registration') if row.get('registration') else None, + manufacturer_icao=row.get('manufacturericao') if row.get('manufacturericao') else None, + type_code=row.get('typecode') if row.get('typecode') else None, + manufacturer_name=row.get('manufacturername') if row.get('manufacturername') else None, + model=row.get('model') if row.get('model') else None + ) + batch.append(aircraft) + loaded += 1 + + # Commit in batches of 1000 + if len(batch) >= 1000: + db.bulk_save_objects(batch) + db.commit() + batch = [] + print(f" Loaded {loaded} aircraft...", end='\r') + + # Commit remaining + if batch: + db.bulk_save_objects(batch) + db.commit() + + print(f" ✓ Loaded {loaded} aircraft records" + " " * 20) + return loaded + + +def main(): + """Main seeding function""" + print("Starting data seeding process...") + + try: + # Create database connection + engine = create_engine(settings.database_url) + Session = sessionmaker(bind=engine) + db = Session() + + # Determine CSV paths - check multiple locations + base_paths = [ + Path('/app/../db-init'), # Docker mounted volume + Path('/app/db-init'), # If copied into container + Path('./db-init'), # Current directory + Path('../db-init'), # Parent directory + ] + + airport_csv = None + aircraft_csv = None + + for base in base_paths: + if base.exists(): + potential_airport = base / 'airports_data_clean.csv' + potential_aircraft = base / 'aircraft_data.csv' + + if potential_airport.exists() and not airport_csv: + airport_csv = str(potential_airport) + if potential_aircraft.exists() and not aircraft_csv: + aircraft_csv = str(potential_aircraft) + + if airport_csv and aircraft_csv: + break + + # Load data + airports_loaded = 0 + aircraft_loaded = 0 + + if airport_csv: + airports_loaded = load_airports(db, airport_csv) + else: + print(" ⚠ No airport CSV file found") + + if aircraft_csv: + aircraft_loaded = load_aircraft(db, aircraft_csv) + else: + print(" ⚠ No aircraft CSV file found") + + db.close() + + print("") + print(f"Seeding complete:") + print(f" Airports: {airports_loaded:,}") + print(f" Aircraft: {aircraft_loaded:,}") + + return 0 + + except Exception as e: + print(f"✗ Error during seeding: {e}") + import traceback + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..8d0cdf5 --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,89 @@ +version: '3.8' + +# Production docker-compose configuration +# This uses an external database and optimized settings + +services: + # FastAPI Backend + api: + build: ./backend + container_name: ppr_prod_api + restart: always + environment: + DB_HOST: ${DB_HOST} + DB_USER: ${DB_USER} + DB_PASSWORD: ${DB_PASSWORD} + DB_NAME: ${DB_NAME} + DB_PORT: ${DB_PORT} + SECRET_KEY: ${SECRET_KEY} + ACCESS_TOKEN_EXPIRE_MINUTES: ${ACCESS_TOKEN_EXPIRE_MINUTES} + API_V1_STR: ${API_V1_STR} + PROJECT_NAME: ${PROJECT_NAME} + MAIL_HOST: ${MAIL_HOST} + MAIL_PORT: ${MAIL_PORT} + MAIL_USERNAME: ${MAIL_USERNAME} + MAIL_PASSWORD: ${MAIL_PASSWORD} + MAIL_FROM: ${MAIL_FROM} + MAIL_FROM_NAME: ${MAIL_FROM_NAME} + BASE_URL: ${BASE_URL} + REDIS_URL: ${REDIS_URL} + ENVIRONMENT: production + WORKERS: "4" + ports: + - "${API_PORT_EXTERNAL}:8000" + volumes: + - ./backend:/app + - ./db-init:/db-init:ro # Mount CSV data for seeding + networks: + - app_network + deploy: + resources: + limits: + cpus: '2' + memory: 2G + reservations: + cpus: '1' + memory: 1G + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 40s + + # Redis for caching (optional) + redis: + image: redis:7-alpine + container_name: ppr_prod_redis + restart: always + networks: + - app_network + deploy: + resources: + limits: + cpus: '0.5' + memory: 512M + + # Nginx web server for public frontend + web: + image: nginx:alpine + container_name: ppr_prod_web + restart: always + ports: + - "${WEB_PORT_EXTERNAL}:80" + volumes: + - ./web:/usr/share/nginx/html:ro + - ./nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + - api + networks: + - app_network + deploy: + resources: + limits: + cpus: '0.5' + memory: 256M + +networks: + app_network: + driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index fc08c66..a921a05 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,10 +44,10 @@ services: - db volumes: - ./backend:/app + - ./db-init:/db-init:ro # Mount CSV data for seeding networks: - private_network - public_network - command: uvicorn app.main:app --host 0.0.0.0 --port 8000 --reload # Redis for caching (optional for now) redis: