From 8f36049cc3281ff9af761fefed4ed010db39c4e7 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 21 Jan 2022 15:56:44 -0800 Subject: web: support for alert messages (site-wide) --- python/fatcat_web/templates/base.html | 14 +++++++++++++- python/fatcat_web/web_config.py | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/python/fatcat_web/templates/base.html b/python/fatcat_web/templates/base.html index 5f507082..23cb5fa7 100644 --- a/python/fatcat_web/templates/base.html +++ b/python/fatcat_web/templates/base.html @@ -45,7 +45,7 @@ -

+ +{% if config.FATCAT_ALERT_MESSAGE %} +

+ + + {{ config.FATCAT_ALERT_MESSAGE | safe }} + + +

+{% endif %} + {% block fullmain %}

diff --git a/python/fatcat_web/web_config.py b/python/fatcat_web/web_config.py index 55d233f3..d58214cc 100644 --- a/python/fatcat_web/web_config.py +++ b/python/fatcat_web/web_config.py @@ -123,6 +123,9 @@ class Config(object): os.environ.get("FATCAT_MERGE_SHADOW_PRESERVATION", default=False) ) + # creates a site-wide alert message + FATCAT_ALERT_MESSAGE = os.environ.get("FATCAT_ALERT_MESSAGE", default=None) or None + # CSRF on by default, but only for WTF forms (not, eg, search, lookups, GET # forms) WTF_CSRF_CHECK_DEFAULT = False -- cgit v1.2.3 From 817b0e76d4b9f5b22e08e90e946af1c14b6cc3e6 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 21 Jan 2022 16:07:03 -0800 Subject: rust: swith postgresql database to 'fatcat_dev' (distinct from 'fatcat_test') --- rust/HACKING.md | 2 +- rust/README.md | 2 +- rust/example.env | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/HACKING.md b/rust/HACKING.md index fbdeb499..997c849d 100644 --- a/rust/HACKING.md +++ b/rust/HACKING.md @@ -50,7 +50,7 @@ Update Rust database schema (after changing raw SQL schema): Debug SQL schema errors (if diesel commands fail): - psql fatcat_test < migrations/2019-01-01-000000_init/up.sql + psql fatcat_dev < migrations/2019-01-01-000000_init/up.sql ## Direct API Interaction diff --git a/rust/README.md b/rust/README.md index 36061240..e9752fa7 100644 --- a/rust/README.md +++ b/rust/README.md @@ -41,7 +41,7 @@ commands from that user. To create the database account: Copy `./example.env` to `./.env` and update the `DATABASE_URL` and `TEST_DATABASE_URL` lines with the password you set above. -As your regular user, use `diesel` to create and initialize the `fatcat` +As your regular user, use `diesel` to create and initialize the `fatcat_dev` database (`diesel` and the fatcat tools will automatically use postgresql credentials from the `.env` file) as well as the `fatcat_test` database for automated tests: diff --git a/rust/example.env b/rust/example.env index 9b59f28d..4ca8a170 100644 --- a/rust/example.env +++ b/rust/example.env @@ -1,4 +1,4 @@ -DATABASE_URL=postgres://fatcat:tactaf@localhost/fatcat +DATABASE_URL=postgres://fatcat:tactaf@localhost/fatcat_dev TEST_DATABASE_URL=postgres://fatcat:tactaf@localhost/fatcat_test AUTH_LOCATION="dev.fatcat.wiki" AUTH_KEY_IDENT="20190101-dev-dummy-key" -- cgit v1.2.3 From c8b2d48106f92215ca2fbd761f8ac4cf254135e6 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 21 Jan 2022 16:42:58 -0800 Subject: rust: handle database read-only mode (as 5xx errors, not 4xx) --- rust/src/endpoints.rs | 14 +++++++------- rust/src/errors.rs | 10 +++++++++- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/rust/src/endpoints.rs b/rust/src/endpoints.rs index e04e7315..1a37ecfe 100644 --- a/rust/src/endpoints.rs +++ b/rust/src/endpoints.rs @@ -36,7 +36,7 @@ macro_rules! generic_auth_err_responses { match $val { NotFound(_, _) | DatabaseRowNotFound => $resp_type::NotFound($val.into()), InvalidCredentials(_) | InsufficientPrivileges(_) => $resp_type::Forbidden($val.into()), - DatabaseError(_) | InternalError(_) => { + DatabaseError(_) | InternalError(_) | DatabaseReadOnly => { error!("{}", $val); capture_fail(&$val); $resp_type::GenericError($val.into()) @@ -51,7 +51,7 @@ macro_rules! generic_err_responses { //use crate::errors::FatcatError::*; match $val { NotFound(_, _) | DatabaseRowNotFound => $resp_type::NotFound($val.into()), - DatabaseError(_) | InternalError(_) => { + DatabaseError(_) | InternalError(_) | DatabaseReadOnly => { error!("{}", $val); capture_fail(&$val); $resp_type::GenericError($val.into()) @@ -1100,7 +1100,7 @@ impl Api for Server { NotFound(_, _) | DatabaseRowNotFound => { CreateEditgroupResponse::NotFound(fe.into()) } - DatabaseError(_) | InternalError(_) => { + DatabaseError(_) | InternalError(_) | DatabaseReadOnly => { error!("{}", fe); capture_fail(&fe); CreateEditgroupResponse::GenericError(fe.into()) @@ -1209,7 +1209,7 @@ impl Api for Server { { Ok(changelog) => GetChangelogResponse::Success(changelog), Err(fe) => match fe { - DatabaseError(_) | InternalError(_) => { + DatabaseError(_) | InternalError(_) | DatabaseReadOnly => { error!("{}", fe); capture_fail(&fe); GetChangelogResponse::GenericError(fe.into()) @@ -1274,7 +1274,7 @@ impl Api for Server { InvalidCredentials(_) | InsufficientPrivileges(_) => { AuthOidcResponse::Forbidden(fe.into()) } - DatabaseError(_) | InternalError(_) => { + DatabaseError(_) | InternalError(_) | DatabaseReadOnly => { error!("{}", fe); capture_fail(&fe); AuthOidcResponse::GenericError(fe.into()) @@ -1322,7 +1322,7 @@ impl Api for Server { InvalidCredentials(_) | InsufficientPrivileges(_) => { AuthCheckResponse::Forbidden(fe.into()) } - DatabaseError(_) | InternalError(_) => { + DatabaseError(_) | InternalError(_) | DatabaseReadOnly => { error!("{}", fe); capture_fail(&fe); AuthCheckResponse::GenericError(fe.into()) @@ -1373,7 +1373,7 @@ impl Api for Server { InvalidCredentials(_) | InsufficientPrivileges(_) => { CreateAuthTokenResponse::Forbidden(fe.into()) } - DatabaseError(_) | InternalError(_) => { + DatabaseError(_) | InternalError(_) | DatabaseReadOnly => { error!("{}", fe); capture_fail(&fe); CreateAuthTokenResponse::GenericError(fe.into()) diff --git a/rust/src/errors.rs b/rust/src/errors.rs index e18e660f..ea0f9646 100644 --- a/rust/src/errors.rs +++ b/rust/src/errors.rs @@ -81,10 +81,13 @@ pub enum FatcatError { // Utf8Decode, StringDecode, Uuid BadRequest(String), - #[fail(display = "unexpected database error: {}", _0)] + #[fail(display = "database error: {}", _0)] // Diesel constraint that we think is a user error ConstraintViolation(String), + #[fail(display = "database in read-only mode (usually replica or maintenance)")] + DatabaseReadOnly, + #[fail(display = "generic database 'not-found'")] // This should generally get caught and handled DatabaseRowNotFound, @@ -117,6 +120,11 @@ impl From for FatcatError { fn from(inner: diesel::result::Error) -> FatcatError { match inner { diesel::result::Error::NotFound => FatcatError::DatabaseRowNotFound, + diesel::result::Error::DatabaseError(_, info) + if info.message().contains("in a read-only transaction") => + { + FatcatError::DatabaseReadOnly + } diesel::result::Error::DatabaseError(_, _) => { FatcatError::ConstraintViolation(inner.to_string()) } -- cgit v1.2.3 From d1c0ae2f9eb0bae6db72ab8995078108c6f7744a Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 21 Jan 2022 16:43:20 -0800 Subject: changelog: read-only mode updates --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32257c1f..f68dc9dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,10 +17,20 @@ See also: ## UNRELEASED +### Changes + +- default local development database now called `fatcat_dev`, not `fatcat` + ### Fixed - various typos and spelling errors corrected (using `codespell`) +### Added + +- "alert banner" mechanism for web interface +- API server detects SQL transaction failure due to read-only mode, and returns + appropriate 5xx error + ## [0.5.0] - 2021-11-22 Small change to the API schema (and SQL schema), adding the `content_scope` -- cgit v1.2.3 From 709c975077615d9af9694784da50a8b4b30e4706 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 21 Jan 2022 16:51:02 -0800 Subject: proposal: read-only database mode --- proposals/2022-01-21_read-only-db.md | 60 ++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 proposals/2022-01-21_read-only-db.md diff --git a/proposals/2022-01-21_read-only-db.md b/proposals/2022-01-21_read-only-db.md new file mode 100644 index 00000000..16d3d166 --- /dev/null +++ b/proposals/2022-01-21_read-only-db.md @@ -0,0 +1,60 @@ + +status: in-progress + +Database Read-Only Mode +======================= + +Fatcat should have the ability to run in "read-only" mode, where all web and +API requests work as usual, except no modifications to the database are +possible. + +There are at least two specific use cases for this mode: + +- during certain maintenance operations, like migrating VMs or upgrading databases +- in the currently-planned high-availability mode, the "replica" VM (database, + API, and web interface) is read-only + +This proposal is to implement this at the SQL database level (using +`default_transaction_read_only` in PostgreSQL), have the API server detect this +error and report appropriately. Additionally, the web interface will have a +configuration variable that allows a site-wide alert message which will display +on every page, which will allow indicating things like upcoming downtime, read-only +mode, and read-replica responses. + +TODO: have fatcat API server handle the specific error when a write is +attempted on a read replica database (distinct from +`default_transaction_read_only`). + + +## Configure PostgreSQL Read-Only Mode + +Connect to database with `psql` (presumably as `postgres` superuser), and run: + + ALTER DATABASE fatcat_prod SET default_transaction_read_only = true; + +Then restart the API server (to force reconnect). + +Revert with the same process, but setting to `false`. + + +## Maintenance Process + +When database maintenance is in process, of a type that requires freezing +writes to the database, first a banner should be set up via +`FATCAT_ALERT_MESSAGE` and restarting `fatcat-web` (see below). + +Then, `default_transaction_read_only=true` can be set via `psql` (see +PostgreSQL section of this doc) and `fatcat-api` restarted. + +When maintenance is done, reverse the process. + + +## Read-Only Backup Web Configuration + +On replica (backup) servers, add a permanent `FATCAT_ALERT_MESSAGE` environment +variable for the `fatcat-web` service, which will impose a loud banner in the +web interface. HTML is allowed, so the message can link elsewhere for more +details. + +If the replica is ever promoted to primary, this environment variable should be +removed and the `fatcat-web` service restarted. -- cgit v1.2.3