Database Commands

# Auto-detect subjects, check one session
neuromaestro check-outputs \
  --project my_study \
  --work /data/processed/my_study \
  --config-dir /data/config \
  --session 01

# Check multiple sessions in one run
neuromaestro check-outputs \
  --project my_study \
  --work /data/processed/my_study \
  --config-dir /data/config \
  --session 01,02

# Filter to specific subjects and tasks
neuromaestro check-outputs \
  --project my_study \
  --work /data/processed/my_study \
  --config-dir /data/config \
  --subjects 001,002,003 \
  --session 01 \
  --task rest_preprocess \
  --task volume

# Project without sessions: pass any value, it is unused by the path templates
neuromaestro check-outputs \
  --project ds027 \
  --work /data/processed/ds027 \
  --config-dir /data/config \
  --session 01

# Use a custom checks directory
neuromaestro check-outputs \
  --project my_study \
  --work /data/processed/my_study \
  --config-dir /data/config \
  --session 01 \
  --checks-dir /path/to/custom/checks/

# work_dir is a positional argument (no flag): the run's --work, without the project name
neuromaestro merge-logs /data/work

# Specify database path explicitly
neuromaestro merge-logs /data/work \
  --db-path /data/work/database/pipeline_jobs.db

Skipped 7 incomplete log(s) with no end event: recon (2), rest_preprocess (5)

# Auto-detect database path from work directory
neuromaestro force-rebuild /data/work

# Specify database path explicitly
neuromaestro force-rebuild /data/work \
  --db-path /data/work/database/pipeline_jobs.db

# The 20 most recent job records (the default limit)
python -m neuromaestro.pipeline.utils.job_db query_jobs \
  --db-path /data/work/database/pipeline_jobs.db

# Failed rest_preprocess jobs only
python -m neuromaestro.pipeline.utils.job_db query_jobs \
  --db-path /data/work/database/pipeline_jobs.db \
  --task-name rest_preprocess \
  --status FAILED

# Minimal: report saved next to the database
neuromaestro generate-report \
  --db-path /data/work/database/pipeline_jobs.db \
  --project my_study \
  --check-results /data/processed/my_study/check_results_20260401_120000.csv

# With --config-dir so tasks are listed in pipeline order
neuromaestro generate-report \
  --db-path /data/work/database/pipeline_jobs.db \
  --project my_study \
  --config-dir /data/config \
  --check-results /data/processed/my_study/check_results_20260401_120000.csv

# Filter by session
neuromaestro generate-report \
  --db-path /data/work/database/pipeline_jobs.db \
  --project my_study \
  --session 01 \
  --check-results /data/processed/my_study/check_results_20260401_120000.csv

# Save to a specific path
neuromaestro generate-report \
  --db-path /data/work/database/pipeline_jobs.db \
  --project my_study \
  --session 01 \
  --check-results /data/processed/my_study/check_results_20260401_120000.csv \
  -o /data/reports/my_study_report.html

-- Find subjects where the script exited non-zero for a task
-- Note: status='FAILED' only catches non-zero exits; silent failures (exit 0, bad output)
-- require check-outputs. Also query for 'CANCELLED' to see SLURM-killed jobs.
SELECT subject, task_name, status, error_msg, start_time
FROM job_status
WHERE task_name = 'rest_preprocess' AND status != 'SUCCESS'
ORDER BY start_time DESC;

-- Jobs from last 7 days
SELECT subject, task_name, status, duration_hours
FROM job_status
WHERE start_time > datetime('now', '-7 days');

-- Full pipeline run history
SELECT execution_time, command_line, total_jobs, status
FROM pipeline_executions
ORDER BY execution_time DESC
LIMIT 10;

-- All jobs submitted in a specific pipeline run
-- (execution_id is the timestamp-based ID printed at submission time)
SELECT j.subject, j.task_name, j.status, j.duration_hours
FROM job_status j
JOIN pipeline_executions p ON j.execution_id = p.execution_id
WHERE p.execution_id = 1699000000000
ORDER BY j.task_name, j.subject;

{--work}/                                    # e.g. /data/work
├── database/
│   ├── pipeline_jobs.db                     # SQLite job database (created by the first merge-logs)
│   ├── backup/
│   │   └── pipeline_jobs.backup_*.db        # Auto-backups (created before each merge-logs)
│   └── json/
│       ├── {task_name}/
│       │   ├── {job_id}_*.jsonl             # Per-job JSONL logs (raw, before merge)
│       │   └── archived/
│       └── _pipeline/
│           ├── execution_*.jsonl            # Pipeline-level event logs
│           ├── wrapper_*.jsonl              # Wrapper script content per submission
│           └── archived/
└── {project}/                               # e.g. /data/work/my_study
    └── log/
        ├── wrapper/
        │   └── {script}_{timestamp}_wrapper.sh   # Generated SLURM scripts, named after the script file
        └── {task_name}/
            ├── {task_name}_{job_id}-{array_index}.out   # SLURM stdout (non-array: {task_name}_{job_id}.out)
            ├── {task_name}_{job_id}-{array_index}.err   # SLURM stderr
            └── sub-{subject}/
                └── {task_name}_{job_id}_{array_index}_{timestamp}.log   # Per-subject execution log