diff --git a/README.md b/README.md index 07842b93..07e9dde1 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,12 @@ These instructions will guide you through setting up the project for local devel - Python 3.11 or later - [uv](https://github.com/astral-sh/uv) - Python package manager -- **Docker and Docker Compose** - Required for running PostgreSQL database +- **Docker and Docker Compose** - Required for running PostgreSQL database and Redis (for async Celery tasks) - Git - Make > **Note:** This application requires PostgreSQL (uses `ArrayField`). Docker is the easiest way to run PostgreSQL locally. - ### 1. Install Dependencies This project uses `uv` to manage dependencies and `poe-the-poet` for task running. First, [install `uv` if you don't have it yet](https://docs.astral.sh/uv/guides/install-python/). @@ -79,25 +78,48 @@ uv run poe bootstrap # Bootstrap data only uv run poe shell # Open the Django shell uv run poe build # Build frontend assets and collect static files uv run poe clean # Remove database volumes (WARNING: destroys all data) +uv run poe celery # Start Celery worker for async task processing +uv run poe celery-dev # Start database and Celery worker ``` +### 4. Async Celery Setup (Optional) + +By default, Celery tasks run synchronously (`CELERY_TASK_ALWAYS_EAGER = True`), which means prediction tasks block the HTTP request until completion. To enable asynchronous task processing with live status updates on pathway pages: + +1. **Set the Celery flag in your `.env` file:** + + ```bash + FLAG_CELERY_PRESENT=True + ``` + +2. **Start Redis and Celery worker:** + + ```bash + uv run poe celery-dev + ``` + +3. **Start the development server** (in another terminal): + ```bash + uv run poe dev + ``` + ### Troubleshooting -* **Docker Connection Error:** If you see an error like `open //./pipe/dockerDesktopLinuxEngine: The system cannot find the file specified` (on Windows), it likely means your Docker Desktop application is not running. Please start Docker Desktop and try the command again. +- **Docker Connection Error:** If you see an error like `open //./pipe/dockerDesktopLinuxEngine: The system cannot find the file specified` (on Windows), it likely means your Docker Desktop application is not running. Please start Docker Desktop and try the command again. -* **SSH Keys for Git Dependencies:** Some dependencies are installed from private git repositories and require SSH authentication. Ensure your SSH keys are configured correctly for Git. - * For a general guide, see [GitHub's official documentation](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent). - * **Windows Users:** If `uv sync` hangs while fetching git dependencies, you may need to explicitly configure Git to use the Windows OpenSSH client and use the `ssh-agent` to manage your key's passphrase. +- **SSH Keys for Git Dependencies:** Some dependencies are installed from private git repositories and require SSH authentication. Ensure your SSH keys are configured correctly for Git. + - For a general guide, see [GitHub's official documentation](https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent). + - **Windows Users:** If `uv sync` hangs while fetching git dependencies, you may need to explicitly configure Git to use the Windows OpenSSH client and use the `ssh-agent` to manage your key's passphrase. + 1. **Point Git to the correct SSH executable:** + ```powershell + git config --global core.sshCommand "C:/Windows/System32/OpenSSH/ssh.exe" + ``` + 2. **Enable and use the SSH agent:** - 1. **Point Git to the correct SSH executable:** - ```powershell - git config --global core.sshCommand "C:/Windows/System32/OpenSSH/ssh.exe" - ``` - 2. **Enable and use the SSH agent:** - ```powershell - # Run these commands in an administrator PowerShell - Get-Service ssh-agent | Set-Service -StartupType Automatic -PassThru | Start-Service + ```powershell + # Run these commands in an administrator PowerShell + Get-Service ssh-agent | Set-Service -StartupType Automatic -PassThru | Start-Service - # Add your key to the agent. It will prompt for the passphrase once. - ssh-add - ``` + # Add your key to the agent. It will prompt for the passphrase once. + ssh-add + ``` diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1e207ad3..0c0cdac6 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,6 +1,6 @@ services: db: - image: postgres:15 + image: postgres:18 container_name: envipath-postgres environment: POSTGRES_USER: postgres @@ -9,12 +9,18 @@ services: ports: - "5432:5432" volumes: - - postgres_data:/var/lib/postgresql/data + - postgres_data:/var/lib/postgresql healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s timeout: 5s retries: 5 + redis: + image: redis:7-alpine + container_name: envipath-redis + ports: + - "6379:6379" + volumes: postgres_data: diff --git a/epdb/legacy_api.py b/epdb/legacy_api.py index c58e651a..747d4ce7 100644 --- a/epdb/legacy_api.py +++ b/epdb/legacy_api.py @@ -1451,6 +1451,7 @@ def create_pathway( setting = SettingManager.get_setting_by_url(request.user, pw.selectedSetting) new_pw.setting = setting + new_pw.kv.update({"status": "running"}) new_pw.save() from .tasks import dispatch, predict diff --git a/epdb/views.py b/epdb/views.py index 4ac4be64..dde70291 100644 --- a/epdb/views.py +++ b/epdb/views.py @@ -1908,6 +1908,7 @@ def package_pathways(request, package_uuid): limit = 1 pw.setting = prediction_setting + pw.kv.update({"status": "running"}) pw.save() from .tasks import dispatch, predict @@ -2059,6 +2060,9 @@ def package_pathway(request, package_uuid, pathway_uuid): if node_url: n = current_pathway.get_node(node_url) + current_pathway.kv.update({"status": "running"}) + current_pathway.save() + from .tasks import dispatch, predict dispatch( diff --git a/pyproject.toml b/pyproject.toml index 0a91c442..8fe196ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -88,6 +88,13 @@ build = { sequence = [ db-up = { cmd = "docker compose -f docker-compose.dev.yml up -d", help = "Start PostgreSQL database using Docker Compose" } db-down = { cmd = "docker compose -f docker-compose.dev.yml down", help = "Stop PostgreSQL database" } +# Celery tasks +celery = { cmd = "celery -A envipath worker -l INFO -Q predict,model,background", help = "Start Celery worker for async task processing" } +celery-dev = { sequence = [ + "db-up", + "celery", +], help = "Start database and Celery worker" } + # Frontend tasks js-deps = { cmd = "uv run python scripts/pnpm_wrapper.py install", help = "Install frontend dependencies" } diff --git a/static/js/alpine/pathway.js b/static/js/alpine/pathway.js index b81de6d1..4c60ea62 100644 --- a/static/js/alpine/pathway.js +++ b/static/js/alpine/pathway.js @@ -21,6 +21,7 @@ document.addEventListener('alpine:init', () => { Alpine.data('pathwayViewer', (config) => ({ status: config.status, modified: config.modified, + modifiedDate: null, statusUrl: config.statusUrl, emptyDueToThreshold: config.emptyDueToThreshold === "True", showUpdateNotice: false, @@ -39,6 +40,8 @@ document.addEventListener('alpine:init', () => { }, init() { + this.modifiedDate = this.parseDate(this.modified); + if (this.status === 'running') { this.startPolling(); } @@ -66,26 +69,39 @@ document.addEventListener('alpine:init', () => { this.showEmptyDueToThresholdNotice = true; } - if (data.modified > this.modified) { - if (!this.emptyDueToThreshold) { - this.showUpdateNotice = true; - this.updateMessage = this.getUpdateMessage(data.status); - } + const nextModifiedDate = this.parseDate(data.modified); + const modifiedChanged = this.hasNewerTimestamp(nextModifiedDate, this.modifiedDate); + const statusChanged = data.status !== this.status; + + if ((modifiedChanged || statusChanged) && !this.emptyDueToThreshold) { + this.showUpdateNotice = true; + this.updateMessage = this.getUpdateMessage(data.status, modifiedChanged, statusChanged); } - if (data.status !== 'running') { - this.status = data.status; - if (this.pollInterval) { - clearInterval(this.pollInterval); - this.pollInterval = null; - } + this.modified = data.modified; + this.modifiedDate = nextModifiedDate; + this.status = data.status; + + if (data.status !== 'running' && this.pollInterval) { + clearInterval(this.pollInterval); + this.pollInterval = null; } } catch (err) { console.error('Polling error:', err); } }, - getUpdateMessage(status) { + getUpdateMessage(status, modifiedChanged, statusChanged) { + // Prefer explicit status change messaging, otherwise fall back to modified change copy + if (statusChanged) { + if (status === 'completed') { + return 'Prediction completed. Reload the page to see the updated Pathway.'; + } + if (status === 'failed') { + return 'Prediction failed. Reload the page to see the latest status.'; + } + } + let msg = 'Prediction '; if (status === 'running') { @@ -99,6 +115,18 @@ document.addEventListener('alpine:init', () => { return msg; }, + parseDate(dateString) { + // Normalize "YYYY-MM-DD HH:mm:ss" into an ISO-compatible string to avoid locale issues + if (!dateString) return null; + return new Date(dateString.replace(' ', 'T')); + }, + + hasNewerTimestamp(nextDate, currentDate) { + if (!nextDate) return false; + if (!currentDate) return true; + return nextDate.getTime() > currentDate.getTime(); + }, + reloadPage() { location.reload(); } diff --git a/templates/components/loading-spinner.html b/templates/components/loading-spinner.html index 9f3cd5f2..8d902988 100644 --- a/templates/components/loading-spinner.html +++ b/templates/components/loading-spinner.html @@ -9,6 +9,11 @@ } .spinner-slow svg { animation: spin-slow 3s linear infinite; + width: 100%; + height: 100%; + transform-origin: center; + transform-box: fill-box; + display: block; }