From 13abeae885b5931d6397e0cac21115d70f906eee Mon Sep 17 00:00:00 2001 From: ryaken-nakamoto Date: Thu, 8 Oct 2026 23:48:50 -0400 Subject: [PATCH 1/2] draft --- README.md | 1 + db/README.md | 88 ++++++++++ db/docker-compose.yml | 22 +++ db/kysely.config.ts | 15 ++ db/{ => migrations}/.gitkeep | 0 db/package.json | 23 +++ db/tsconfig.json | 12 ++ db/yarn.lock | 300 +++++++++++++++++++++++++++++++++++ 8 files changed, 461 insertions(+) create mode 100644 db/README.md create mode 100644 db/docker-compose.yml create mode 100644 db/kysely.config.ts rename db/{ => migrations}/.gitkeep (100%) create mode 100644 db/package.json create mode 100644 db/tsconfig.json create mode 100644 db/yarn.lock diff --git a/README.md b/README.md index e287df179..9b0e2bb1d 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ The backend is **Lambda-only**. Four independent Lambda functions live under `ap - `ingestion-lambda` — EventBridge-scheduled, pulls live conditions from the Davis weather station - `public-lambda` — serve the public dashboard +For a local Postgres and database migrations, see [db/README.md](db/README.md). ## Running tasks diff --git a/db/README.md b/db/README.md new file mode 100644 index 000000000..ddc199bd3 --- /dev/null +++ b/db/README.md @@ -0,0 +1,88 @@ +# Local database + +A disposable Postgres 16 container plus [kysely-ctl](https://github.com/kysely-org/kysely-ctl), the Kysely migration CLI. The schema here is shared by all the backend Lambdas. + +## Prerequisites + +- Node 24 (`nvm use` at the repo root). `kysely-ctl` needs Node 22 or later, and `yarn` fails on older versions. +- A running Docker engine (Docker Desktop, Colima, or OrbStack) with Compose v2.1 or later (`docker compose version`) + +## Quick start + +Run everything from this `db/` folder, not the repo root. + +```bash +cd db +yarn +yarn db:up +yarn db:migrate +``` + +With no migrations yet, `yarn db:migrate` prints `Migration skipped: no new migrations found` and exits 0. The `Ignoring .gitkeep` warning is harmless. + +## Commands + +| Command | What it does | +| --- | --- | +| `yarn db:up` | Start Postgres and wait until it is healthy | +| `yarn db:down` | Stop the container, keep the data | +| `yarn db:migrate` | Apply all pending migrations | +| `yarn db:make ` | Create a new timestamped migration file | +| `yarn db:reset` | Delete the container and data, start fresh, re-run migrations | +| `yarn kysely migrate:list` | Show which migrations have been applied | + +## Connection settings + +Defaults work out of the box. Override with environment variables: + +| Variable | Default | +| --- | --- | +| `DB_HOST` | `127.0.0.1` | +| `DB_PORT` | `5432` | +| `DB_USER` | `postgres` | +| `DB_PASSWORD` | `password` | +| `DB_NAME` | `bho` | + +Connect with psql: `docker compose exec postgres psql -U postgres -d bho` + +## Adding a migration + +Run `yarn db:make create_stations`. It creates `migrations/_create_stations.ts` with empty `up` and `down` functions. The timestamp prefix keeps migrations from different branches from colliding, and files run in filename order. Fill in the functions, for example: + +```ts +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('stations') + .addColumn('id', 'serial', (col) => col.primaryKey()) + .addColumn('name', 'text', (col) => col.notNull()) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropTable('stations').execute(); +} +``` + +Then run `yarn db:migrate`. Applied migrations are recorded in the `kysely_migration` table, so a second run does nothing. + +## How this differs from RDS + +- Disposable: no backups, no Multi-AZ, no failover. +- No TLS and no IAM auth. The default password is weak, so the port is bound to `127.0.0.1` only. +- `yarn db:reset` destroys all data. Local use only. +- Keep the Postgres major version in `docker-compose.yml` (16) in line with the RDS instance. + +## Troubleshooting + +- **`Cannot connect to the Docker daemon`**: start Docker Desktop, then retry. +- **`address already in use` / port 5432 busy**: another Postgres is running on your machine. Find it with `sudo lsof -nP -iTCP:5432 -sTCP:LISTEN`, or use another port: `DB_PORT=5433 yarn db:up` and `DB_PORT=5433 yarn db:migrate`. +- **`yarn db:migrate` fails with `ECONNREFUSED`**: the container is not running. Run `yarn db:up`. +- **`password authentication failed`**: you are connected to a different Postgres than the container, usually because of the port conflict above. +- **`corrupted migrations: previously executed migration … is missing`**: you deleted or renamed a migration file that was already applied to your local database (for example after switching branches). Run `yarn db:reset`. +- **`engine "node" is incompatible`**: run `nvm use` to switch to Node 24. + +## Note on workspaces + +This folder has its own `package.json` and `yarn.lock` because the repo root does not use Yarn workspaces. If workspaces are adopted, add `db/` to them and drop this lockfile. diff --git a/db/docker-compose.yml b/db/docker-compose.yml new file mode 100644 index 000000000..f5b2eeb95 --- /dev/null +++ b/db/docker-compose.yml @@ -0,0 +1,22 @@ +# Disposable local Postgres for development only. +# `yarn db:reset` deletes the volume and every row in it. Never point real data at this. +services: + postgres: + image: postgres:16-alpine + container_name: bho-postgres + environment: + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-password} + POSTGRES_DB: ${DB_NAME:-bho} + ports: + - '127.0.0.1:${DB_PORT:-5432}:5432' + volumes: + - bho-pgdata:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}'] + interval: 2s + timeout: 3s + retries: 15 + +volumes: + bho-pgdata: diff --git a/db/kysely.config.ts b/db/kysely.config.ts new file mode 100644 index 000000000..39ed13603 --- /dev/null +++ b/db/kysely.config.ts @@ -0,0 +1,15 @@ +import { PostgresDialect } from 'kysely'; +import { defineConfig } from 'kysely-ctl'; +import { Pool } from 'pg'; + +export default defineConfig({ + dialect: new PostgresDialect({ + pool: new Pool({ + host: process.env.DB_HOST ?? '127.0.0.1', + port: Number(process.env.DB_PORT ?? 5432), + user: process.env.DB_USER ?? 'postgres', + password: process.env.DB_PASSWORD ?? 'password', + database: process.env.DB_NAME ?? 'bho', + }), + }), +}); diff --git a/db/.gitkeep b/db/migrations/.gitkeep similarity index 100% rename from db/.gitkeep rename to db/migrations/.gitkeep diff --git a/db/package.json b/db/package.json new file mode 100644 index 000000000..109090d2a --- /dev/null +++ b/db/package.json @@ -0,0 +1,23 @@ +{ + "name": "@bho/db", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "db:up": "docker compose up -d --wait", + "db:down": "docker compose down", + "db:migrate": "kysely migrate:latest", + "db:make": "kysely migrate:make", + "db:reset": "docker compose down -v && yarn db:up && yarn db:migrate" + }, + "dependencies": { + "kysely": "^0.27.0", + "pg": "^8.11.0" + }, + "devDependencies": { + "@types/node": "^20.0.0", + "@types/pg": "^8.11.0", + "kysely-ctl": "^0.21.0", + "typescript": "^5.4.0" + } +} diff --git a/db/tsconfig.json b/db/tsconfig.json new file mode 100644 index 000000000..6b37cf00c --- /dev/null +++ b/db/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "module": "nodenext", + "moduleResolution": "nodenext", + "target": "es2022", + "esModuleInterop": true, + "strict": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["*.ts", "migrations/**/*.ts"] +} diff --git a/db/yarn.lock b/db/yarn.lock new file mode 100644 index 000000000..b3607cda7 --- /dev/null +++ b/db/yarn.lock @@ -0,0 +1,300 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@types/node@*": + version "26.6.4" + resolved "https://registry.yarnpkg.com/@types/node/-/node-26.6.4.tgz#5aaac1a7337945f7f9a2de4e0d14e7107157e813" + integrity sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg== + dependencies: + undici-types "~8.9.0" + +"@types/node@^20.0.0": + version "20.19.43" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.19.43.tgz#fcecf580ba42a0db55cf404c372c97973c376c97" + integrity sha512-6oYBAi5ikg4Pl+kGsoYtawUMBT2zZMCvPNF7pVLnHZfd1zf38DRiWn/gT01RYCdUqkv7Fhr+C9ot4/tb+2sVvA== + dependencies: + undici-types "~6.21.0" + +"@types/pg@^8.11.0": + version "8.23.1" + resolved "https://registry.yarnpkg.com/@types/pg/-/pg-8.23.1.tgz#7e712ce02cf44ad100862127f10f52e9c6b9d227" + integrity sha512-fKVHpikPdg4GKks3JuLEhvwSyvwzF23hnabPy6DD8ljVbC7+6J5dQzdv4arV6jqq57djnMgs1HKBxX4P8aBI3A== + dependencies: + "@types/node" "*" + pg-protocol "*" + pg-types "^2.2.0" + +c12@^3.3.3: + version "3.3.4" + resolved "https://registry.yarnpkg.com/c12/-/c12-3.3.4.tgz#1253a5faf8b61244884d42459b4a6412571fe9f3" + integrity sha512-cM0ApFQSBXuourJejzwv/AuPRvAxordTyParRVcHjjtXirtkzM0uK2L9TTn9s0cXZbG7E55jCivRQzoxYmRAlA== + dependencies: + chokidar "^5.0.0" + confbox "^0.2.4" + defu "^6.1.6" + dotenv "^17.3.1" + exsolve "^1.0.8" + giget "^3.2.0" + jiti "^2.6.1" + ohash "^2.0.11" + pathe "^2.0.3" + perfect-debounce "^2.1.0" + pkg-types "^2.3.0" + rc9 "^3.0.1" + +chokidar@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-5.0.0.tgz#949c126a9238a80792be9a0265934f098af369a5" + integrity sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw== + dependencies: + readdirp "^5.0.0" + +citty@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/citty/-/citty-0.2.2.tgz#92d3f7d13868a730ab06c420bb10bded06cf259f" + integrity sha512-+6vJA3L98yv+IdfKGZHBNiGW5KHn22e/JwID0Strsz8h4S/csAu/OuICwxrg44k5MRiZHWIo8XXuJgQTriRP4w== + +confbox@^0.2.4: + version "0.2.4" + resolved "https://registry.yarnpkg.com/confbox/-/confbox-0.2.4.tgz#592e7be71f882a4a874e3c88f0ac1ef6f7da1ce5" + integrity sha512-ysOGlgTFbN2/Y6Cg3Iye8YKulHw+R2fNXHrgSmXISQdMnomY6eNDprVdW9R5xBguEqI954+S6709UyiO7B+6OQ== + +confbox@^0.3.1: + version "0.3.1" + resolved "https://registry.yarnpkg.com/confbox/-/confbox-0.3.1.tgz#79aa91893791a202fb349fb367e7320423f7c76e" + integrity sha512-cKUSoKa8YxFZZSmraVi7onONx3amu77ngK3kGpsYHDH7drPwCRkQE1RYMPlLRrMtnciRj274XNRxcHxnKmDSnA== + +consola@^3.4.2: + version "3.4.2" + resolved "https://registry.yarnpkg.com/consola/-/consola-3.4.2.tgz#5af110145397bb67afdab77013fdc34cae590ea7" + integrity sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA== + +defu@^6.1.6, defu@^6.1.7: + version "6.1.7" + resolved "https://registry.yarnpkg.com/defu/-/defu-6.1.7.tgz#72543567c8e9f97ff13ce402b6dbe09ac5ae4d23" + integrity sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ== + +destr@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/destr/-/destr-2.0.5.tgz#7d112ff1b925fb8d2079fac5bdb4a90973b51fdb" + integrity sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA== + +dotenv@^17.3.1: + version "17.4.2" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-17.4.2.tgz#c07e54a746e11eba021dd9e1047ced5afdc1c034" + integrity sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw== + +exsolve@^1.0.8, exsolve@^1.1.1: + version "1.1.3" + resolved "https://registry.yarnpkg.com/exsolve/-/exsolve-1.1.3.tgz#175a6336d6cb1c9cfee822a092665ee0b79e716d" + integrity sha512-kuzL9zherLQoqDeQaS55AiiIg1kb4KYXaSusTtgZTPlvKP2gGCI82LNmBNh5RjDVUXIcmgx+/Mx7vPwRzBbhJA== + +giget@^3.2.0: + version "3.3.1" + resolved "https://registry.yarnpkg.com/giget/-/giget-3.3.1.tgz#4a4e610cd112e5dc478c6035986fdc19c76dad73" + integrity sha512-r+mvuDjrjMpsdw46Kmeydb8bdHm7wOKw8wNBtTndkjbPjgAp5oUJUxRE76wZFknxIPokfWvep2qSXK37aXE6zg== + +jiti@^2.6.1, jiti@^2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/jiti/-/jiti-2.7.0.tgz#974228f2f4ca2bc21885a1797b45fea68e950c64" + integrity sha512-AC/7JofJvZGrrneWNaEnJeOLUx+JlGt7tNa0wZiRPT4MY1wmfKjt2+6O2p2uz2+skll8OZZmJMNqeke7kKbNgQ== + +kysely-ctl@^0.21.0: + version "0.21.0" + resolved "https://registry.yarnpkg.com/kysely-ctl/-/kysely-ctl-0.21.0.tgz#e1cdf8a983e0fce2387e721fb35c45b8709e6e4e" + integrity sha512-tovLEh+5eHgBt1P9L42Y3lgEI7cXK0nAPIBFg+dmTXUlmb2Ats3X3D0TwUtCV63XzVEdRV5dONa8DBhRiEf8fg== + dependencies: + c12 "^3.3.3" + citty "^0.2.2" + confbox "^0.2.4" + consola "^3.4.2" + jiti "^2.7.0" + nypm "^0.6.6" + ofetch "^1.5.1" + pathe "^2.0.3" + pkg-types "^2.3.1" + std-env "^4.1.0" + +kysely@^0.27.0: + version "0.27.6" + resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.27.6.tgz#b2a908e160e510f9945586ca9de1616c187a2a8e" + integrity sha512-FIyV/64EkKhJmjgC0g2hygpBv5RNWVPyNCqSAD7eTCv6eFWNIi4PN1UvdSJGicN/o35bnevgis4Y0UDC0qi8jQ== + +node-fetch-native@^1.6.7: + version "1.6.7" + resolved "https://registry.yarnpkg.com/node-fetch-native/-/node-fetch-native-1.6.7.tgz#9d09ca63066cc48423211ed4caf5d70075d76a71" + integrity sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q== + +nypm@^0.6.6: + version "0.6.10" + resolved "https://registry.yarnpkg.com/nypm/-/nypm-0.6.10.tgz#1e2f6881980b62c421c4796c94262793c273d38e" + integrity sha512-W72Hrj1petq+b3Hk2aAC+9zswetlIFTnDW4s5djseZh2nYqBbyQLOtj472HwcbcWykPBUW1WpWpjOd4nK6gMRw== + dependencies: + citty "^0.2.2" + pathe "^2.0.3" + tinyexec "^1.3.1" + +ofetch@^1.5.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/ofetch/-/ofetch-1.5.1.tgz#5c43cc56e03398b273014957060344254505c5c7" + integrity sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA== + dependencies: + destr "^2.0.5" + node-fetch-native "^1.6.7" + ufo "^1.6.1" + +ohash@^2.0.11: + version "2.0.12" + resolved "https://registry.yarnpkg.com/ohash/-/ohash-2.0.12.tgz#cb97c19888ff31e7b22214cd7924187fc97ef05c" + integrity sha512-65S/5gk9YSsaRjcyf7Nfa6h/d3E8/1gslpXfI4W7Dxn/oap8IKRuNT5VXkLQ1YFKIEg4apRY4Pj6aiwFzrDdmw== + +pathe@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/pathe/-/pathe-2.0.3.tgz#3ecbec55421685b70a9da872b2cff3e1cbed1716" + integrity sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w== + +perfect-debounce@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/perfect-debounce/-/perfect-debounce-2.1.0.tgz#e7078e38f231cb191855c3136a4423aef725d261" + integrity sha512-LjgdTytVFXeUgtHZr9WYViYSM/g8MkcTPYDlPa3cDqMirHjKiSZPYd6DoL7pK8AJQr+uWkQvCjHNdiMqsrJs+g== + +pg-cloudflare@^1.4.1: + version "1.4.1" + resolved "https://registry.yarnpkg.com/pg-cloudflare/-/pg-cloudflare-1.4.1.tgz#0fc904c9a8b3ca3a4f71e16bcc9171224ecbb2a8" + integrity sha512-6PQbsFWZcp9EmJEwy5cGQ2La+AMWpP46lgbb8X+U/XsHIUweYDNCpeuKck5RxL2MdVFi7krbbEi5nX4Zh7JhrQ== + +pg-connection-string@^2.14.1: + version "2.14.1" + resolved "https://registry.yarnpkg.com/pg-connection-string/-/pg-connection-string-2.14.1.tgz#d9529fe2497c0cac8a9f5f6f059679c52f1bd964" + integrity sha512-qR3kGNPBLpCNtz0evbKA0Y/MRFXwSSdT+pTJvYp/bXTcReZbvX1kzF0IyTc1QnxqF7AZbOeBhNL8R5mYQZV/MA== + +pg-int8@1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/pg-int8/-/pg-int8-1.0.1.tgz#943bd463bf5b71b4170115f80f8efc9a0c0eb78c" + integrity sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw== + +pg-pool@^3.14.0: + version "3.14.0" + resolved "https://registry.yarnpkg.com/pg-pool/-/pg-pool-3.14.0.tgz#f35ae4eb846780cad71af24099b3edfa9781ad90" + integrity sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw== + +pg-protocol@*, pg-protocol@^1.16.1: + version "1.16.1" + resolved "https://registry.yarnpkg.com/pg-protocol/-/pg-protocol-1.16.1.tgz#87a3951b57002682d63166c029859bd834e64bf1" + integrity sha512-p9VOFMiHB/ZbJATetbg+99PxssTVSQRnyuPSQ67mN1+1KBOjZaZ83ZQzltnxPhJwSsC3nwVjJ10DVJlerbFzLg== + +pg-types@2.2.0, pg-types@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/pg-types/-/pg-types-2.2.0.tgz#2d0250d636454f7cfa3b6ae0382fdfa8063254a3" + integrity sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA== + dependencies: + pg-int8 "1.0.1" + postgres-array "~2.0.0" + postgres-bytea "~1.0.0" + postgres-date "~1.0.4" + postgres-interval "^1.1.0" + +pg@^8.11.0: + version "8.23.1" + resolved "https://registry.yarnpkg.com/pg/-/pg-8.23.1.tgz#e4848051a04aa2f968fd5671d075f89e5e7918b9" + integrity sha512-aL96AHANtWjPLDOLqnhx+ngp9+UK7ETEU8VJrDCGvsSSi/mGLcWYsS6Herg7lmaBJe4uwrfqsa7gTEFaSizDoQ== + dependencies: + pg-connection-string "^2.14.1" + pg-pool "^3.14.0" + pg-protocol "^1.16.1" + pg-types "2.2.0" + pgpass "1.0.5" + optionalDependencies: + pg-cloudflare "^1.4.1" + +pgpass@1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/pgpass/-/pgpass-1.0.5.tgz#9b873e4a564bb10fa7a7dbd55312728d422a223d" + integrity sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug== + dependencies: + split2 "^4.1.0" + +pkg-types@^2.3.0, pkg-types@^2.3.1: + version "2.3.3" + resolved "https://registry.yarnpkg.com/pkg-types/-/pkg-types-2.3.3.tgz#7fa477b1050acf906fdb8896bb2accd52c45e3d5" + integrity sha512-j/lCFdcppV0JxWpCEITdbDltBxPP6cHT+yNJ6Go2OgoSA9518X847X9z0p6LtA4Nc16+eQzCZjRrWanTGvHJ5w== + dependencies: + confbox "^0.3.1" + exsolve "^1.1.1" + pathe "^2.0.3" + +postgres-array@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" + integrity sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA== + +postgres-bytea@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/postgres-bytea/-/postgres-bytea-1.0.1.tgz#c40b3da0222c500ff1e51c5d7014b60b79697c7a" + integrity sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ== + +postgres-date@~1.0.4: + version "1.0.7" + resolved "https://registry.yarnpkg.com/postgres-date/-/postgres-date-1.0.7.tgz#51bc086006005e5061c591cee727f2531bf641a8" + integrity sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q== + +postgres-interval@^1.1.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/postgres-interval/-/postgres-interval-1.2.0.tgz#b460c82cb1587507788819a06aa0fffdb3544695" + integrity sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ== + dependencies: + xtend "^4.0.0" + +rc9@^3.0.1: + version "3.1.0" + resolved "https://registry.yarnpkg.com/rc9/-/rc9-3.1.0.tgz#383f44c0f0bdefcf411c1ce9403c528064a8bd1a" + integrity sha512-ufjkNVzbRHKcCOmTahZkmVsyc3W+MSk3jY03m+a7tGHkIsdVMG9l10/3HvFbWkkKzY5VFp3pkRsIo/UYgmFL7Q== + dependencies: + defu "^6.1.7" + destr "^2.0.5" + +readdirp@^5.0.0: + version "5.1.1" + resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-5.1.1.tgz#520bca06f9d1ae1b96cc0800dbe84b983d19422c" + integrity sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA== + +split2@^4.1.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/split2/-/split2-4.2.0.tgz#c9c5920904d148bab0b9f67145f245a86aadbfa4" + integrity sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg== + +std-env@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/std-env/-/std-env-4.3.0.tgz#4e7231b61fd83219aa7f76f71fa8818b07e41321" + integrity sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag== + +tinyexec@^1.3.1: + version "1.3.1" + resolved "https://registry.yarnpkg.com/tinyexec/-/tinyexec-1.3.1.tgz#16a2e3c6e23fafce72640e678e651db36145b9c6" + integrity sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA== + +typescript@^5.4.0: + version "5.9.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.9.3.tgz#5b4f59e15310ab17a216f5d6cf53ee476ede670f" + integrity sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw== + +ufo@^1.6.1: + version "1.6.4" + resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.6.4.tgz#7a8fb875fcc6382d2c7d0b3692738b0500a92467" + integrity sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA== + +undici-types@~6.21.0: + version "6.21.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.21.0.tgz#691d00af3909be93a7faa13be61b3a5b50ef12cb" + integrity sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ== + +undici-types@~8.9.0: + version "8.9.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-8.9.0.tgz#e240d97c8b5d85e5347ce73d25865c7906c1ec9f" + integrity sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg== + +xtend@^4.0.0: + version "4.0.2" + resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" + integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== From 4dded585d00e9e1b7a9aeb3fe07829b48213f290 Mon Sep 17 00:00:00 2001 From: ryaken-nakamoto Date: Sun, 11 Oct 2026 20:49:46 -0400 Subject: [PATCH 2/2] more concise --- README.md | 14 +++++++- db/README.md | 83 +++++++++++++++---------------------------- db/docker-compose.yml | 22 ++++-------- db/kysely.config.ts | 2 +- db/project.json | 32 +++++++++++++++++ 5 files changed, 81 insertions(+), 72 deletions(-) create mode 100644 db/project.json diff --git a/README.md b/README.md index 9b0e2bb1d..a71cb1f02 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,19 @@ The backend is **Lambda-only**. Four independent Lambda functions live under `ap - `ingestion-lambda` — EventBridge-scheduled, pulls live conditions from the Davis weather station - `public-lambda` — serve the public dashboard -For a local Postgres and database migrations, see [db/README.md](db/README.md). +### Database + +The data lives in PostgreSQL. The schema is shared by all the Lambdas and is defined by [Kysely](https://kysely.dev) migrations in `db/`. For local development, `db/` also provides a disposable Postgres container. + +```bash +nx run db:install # one-time install of db/'s dependencies +nx run db:up # start the local Postgres container (needs Docker) +nx run db:migrate # apply migrations +nx run db:make # create a new migration +nx run db:reset # wipe the local database and re-apply everything +``` + +See [db/README.md](db/README.md) for details, writing migrations, and troubleshooting. ## Running tasks diff --git a/db/README.md b/db/README.md index ddc199bd3..6f6fb8b2d 100644 --- a/db/README.md +++ b/db/README.md @@ -1,88 +1,63 @@ # Local database -A disposable Postgres 16 container plus [kysely-ctl](https://github.com/kysely-org/kysely-ctl), the Kysely migration CLI. The schema here is shared by all the backend Lambdas. +A disposable Postgres 16 container and [kysely-ctl](https://github.com/kysely-org/kysely-ctl) for migrations. The schema is shared by all the backend Lambdas. -## Prerequisites +## Setup -- Node 24 (`nvm use` at the repo root). `kysely-ctl` needs Node 22 or later, and `yarn` fails on older versions. -- A running Docker engine (Docker Desktop, Colima, or OrbStack) with Compose v2.1 or later (`docker compose version`) - -## Quick start - -Run everything from this `db/` folder, not the repo root. +Requires Node 24 (`nvm use` at the repo root) and a running Docker engine with Compose v2. ```bash cd db -yarn -yarn db:up -yarn db:migrate +yarn install # install this folder's dependencies +yarn db:up # start Postgres on 127.0.0.1:5432 +yarn db:migrate # apply migrations ``` -With no migrations yet, `yarn db:migrate` prints `Migration skipped: no new migrations found` and exits 0. The `Ignoring .gitkeep` warning is harmless. +Connection: user `postgres`, password `password`, `127.0.0.1:5432`, database `postgres`. ## Commands -| Command | What it does | -| --- | --- | -| `yarn db:up` | Start Postgres and wait until it is healthy | -| `yarn db:down` | Stop the container, keep the data | -| `yarn db:migrate` | Apply all pending migrations | -| `yarn db:make ` | Create a new timestamped migration file | -| `yarn db:reset` | Delete the container and data, start fresh, re-run migrations | -| `yarn kysely migrate:list` | Show which migrations have been applied | +| Command | What it does | +| --------------------- | ----------------------------------------------- | +| `yarn db:up` | Start Postgres | +| `yarn db:down` | Stop it, keep the data | +| `yarn db:migrate` | Apply pending migrations | +| `yarn db:make ` | Create `migrations/_.ts` | +| `yarn db:reset` | Delete the data, start fresh, re-run migrations | -## Connection settings +The same commands run from the repo root through Nx: `nx run db:install`, `nx run db:up`, `nx run db:migrate`, `nx run db:make `, `nx run db:reset` (prefix with `npx` if `nx` isn't installed globally). -Defaults work out of the box. Override with environment variables: +## Adding a migration -| Variable | Default | -| --- | --- | -| `DB_HOST` | `127.0.0.1` | -| `DB_PORT` | `5432` | -| `DB_USER` | `postgres` | -| `DB_PASSWORD` | `password` | -| `DB_NAME` | `bho` | +The database schema is defined by migrations: the schema is whatever all the migration files produce when run in order (oldest first). To change it, add a new migration file. -Connect with psql: `docker compose exec postgres psql -U postgres -d bho` +Each migration exports an **`up`** function that applies the change: create a table, add a column, add an index. `yarn db:migrate` runs `up` for every migration that hasn't been applied yet. Kysely records the applied ones in a `kysely_migration` table, so each runs once. -## Adding a migration +We are forward-only: don't write a `down` function (the one that would undo `up`). Leave the generated `down` stub empty (we don't need to support rollback in dev). For local dev we can just drop the database. To undo or fix a change that has been merged, add a new migration. -Run `yarn db:make create_stations`. It creates `migrations/_create_stations.ts` with empty `up` and `down` functions. The timestamp prefix keeps migrations from different branches from colliding, and files run in filename order. Fill in the functions, for example: +Run `yarn db:make create_stations`, then fill in `up` in the generated file: ```ts -import { Kysely } from 'kysely'; +import type { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema .createTable('stations') .addColumn('id', 'serial', (col) => col.primaryKey()) - .addColumn('name', 'text', (col) => col.notNull()) .execute(); } - -export async function down(db: Kysely): Promise { - await db.schema.dropTable('stations').execute(); -} ``` -Then run `yarn db:migrate`. Applied migrations are recorded in the `kysely_migration` table, so a second run does nothing. +Never edit a migration that has been merged; add a new one. For example, to add a column to `stations` later, create a new migration whose `up` is `alterTable('stations').addColumn(...)`. -## How this differs from RDS +## Differs from RDS -- Disposable: no backups, no Multi-AZ, no failover. -- No TLS and no IAM auth. The default password is weak, so the port is bound to `127.0.0.1` only. -- `yarn db:reset` destroys all data. Local use only. -- Keep the Postgres major version in `docker-compose.yml` (16) in line with the RDS instance. +Disposable: no backups - `yarn db:reset` deletes everything. This is bound to `127.0.0.1` only. ## Troubleshooting -- **`Cannot connect to the Docker daemon`**: start Docker Desktop, then retry. -- **`address already in use` / port 5432 busy**: another Postgres is running on your machine. Find it with `sudo lsof -nP -iTCP:5432 -sTCP:LISTEN`, or use another port: `DB_PORT=5433 yarn db:up` and `DB_PORT=5433 yarn db:migrate`. -- **`yarn db:migrate` fails with `ECONNREFUSED`**: the container is not running. Run `yarn db:up`. -- **`password authentication failed`**: you are connected to a different Postgres than the container, usually because of the port conflict above. -- **`corrupted migrations: previously executed migration … is missing`**: you deleted or renamed a migration file that was already applied to your local database (for example after switching branches). Run `yarn db:reset`. -- **`engine "node" is incompatible`**: run `nvm use` to switch to Node 24. - -## Note on workspaces - -This folder has its own `package.json` and `yarn.lock` because the repo root does not use Yarn workspaces. If workspaces are adopted, add `db/` to them and drop this lockfile. +- **Cannot connect to the Docker daemon:** start Docker Desktop. +- **Port 5432 already in use:** another Postgres is running. Find it with `sudo lsof -nP -iTCP:5432 -sTCP:LISTEN` and stop it. +- **`ECONNREFUSED` right after `db:up`:** Postgres is still starting. Wait a few seconds and rerun. +- **`corrupted migrations … is missing`:** a migration file that was already applied is gone (for example after switching branches). Run `yarn db:reset`. +- **`engine "node" is incompatible`:** run `nvm use`. diff --git a/db/docker-compose.yml b/db/docker-compose.yml index f5b2eeb95..1e7f57944 100644 --- a/db/docker-compose.yml +++ b/db/docker-compose.yml @@ -1,22 +1,12 @@ -# Disposable local Postgres for development only. -# `yarn db:reset` deletes the volume and every row in it. Never point real data at this. services: - postgres: + db: image: postgres:16-alpine - container_name: bho-postgres - environment: - POSTGRES_USER: ${DB_USER:-postgres} - POSTGRES_PASSWORD: ${DB_PASSWORD:-password} - POSTGRES_DB: ${DB_NAME:-bho} ports: - - '127.0.0.1:${DB_PORT:-5432}:5432' + - "127.0.0.1:5432:5432" + environment: + POSTGRES_PASSWORD: password volumes: - - bho-pgdata:/var/lib/postgresql/data - healthcheck: - test: ['CMD-SHELL', 'pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}'] - interval: 2s - timeout: 3s - retries: 15 + - db_data:/var/lib/postgresql/data volumes: - bho-pgdata: + db_data: \ No newline at end of file diff --git a/db/kysely.config.ts b/db/kysely.config.ts index 39ed13603..6a1c7a550 100644 --- a/db/kysely.config.ts +++ b/db/kysely.config.ts @@ -9,7 +9,7 @@ export default defineConfig({ port: Number(process.env.DB_PORT ?? 5432), user: process.env.DB_USER ?? 'postgres', password: process.env.DB_PASSWORD ?? 'password', - database: process.env.DB_NAME ?? 'bho', + database: process.env.DB_NAME ?? 'postgres', }), }), }); diff --git a/db/project.json b/db/project.json new file mode 100644 index 000000000..b72ecbf0e --- /dev/null +++ b/db/project.json @@ -0,0 +1,32 @@ +{ + "name": "db", + "$schema": "../node_modules/nx/schemas/project-schema.json", + "projectType": "application", + "targets": { + "install": { + "executor": "nx:run-commands", + "options": { "command": "yarn install --frozen-lockfile", "cwd": "db" } + }, + "up": { + "executor": "nx:run-commands", + "options": { "command": "yarn db:up", "cwd": "db" } + }, + "down": { + "executor": "nx:run-commands", + "options": { "command": "yarn db:down", "cwd": "db" } + }, + "migrate": { + "executor": "nx:run-commands", + "options": { "command": "yarn db:migrate", "cwd": "db" } + }, + "make": { + "executor": "nx:run-commands", + "options": { "command": "yarn db:make", "cwd": "db" } + }, + "reset": { + "executor": "nx:run-commands", + "options": { "command": "yarn db:reset", "cwd": "db" } + } + }, + "tags": [] +}