From 705ac0fba2dc879115509006ce4fdfae8c9fc5c0 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Fri, 15 Mar 2019 15:39:59 -0700 Subject: update dev setup instructions --- .gitlab-ci.yml | 4 ++-- python/README.md | 2 +- python/env.example | 20 -------------------- python/example.env | 20 ++++++++++++++++++++ python/tests/fixtures.py | 4 ++-- rust/INSTALL.md | 7 +++---- rust/README.md | 32 +++++++++++++++++++++----------- rust/env.example | 10 ---------- rust/example.env | 10 ++++++++++ 9 files changed, 59 insertions(+), 50 deletions(-) delete mode 100644 python/env.example create mode 100644 python/example.env delete mode 100644 rust/env.example create mode 100644 rust/example.env diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6cde9663..e9ee0ba4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,7 @@ unified_test: script: - rustc --version && cargo --version && diesel --version - cd rust - - cp env.example .env + - cp example.env .env - diesel database reset && diesel migration run - cargo build - cargo test -- --test-threads 1 @@ -28,7 +28,7 @@ unified_test: - cd ../python_client - pytest-3 - cd ../python - - cp env.example .env + - cp example.env .env - pipenv install --dev --deploy - pipenv run pytest --cov - pipenv run ./tests/cli.sh diff --git a/python/README.md b/python/README.md index 2760885c..fe9a411d 100644 --- a/python/README.md +++ b/python/README.md @@ -34,7 +34,7 @@ server on the same machine by default), use: # will listen on http://localhost:9810 by default pipenv run fatcat_webface.py -Almost all configuration is done via environment variables; see `env.example` +Almost all configuration is done via environment variables; see `example.env` for a list of settings. If you copy this file to `.env` it will be sourced by `pipenv` automatically; you can also load it in your shell like `source .env`. diff --git a/python/env.example b/python/env.example deleted file mode 100644 index 4bf76e21..00000000 --- a/python/env.example +++ /dev/null @@ -1,20 +0,0 @@ -FLASK_SECRET_KEY="TODO-REPLACE-ME" -FATCAT_DOMAIN="dev.fatcat.wiki" -# This key used in tests -FATCAT_API_AUTH_TOKEN="AgEPZGV2LmZhdGNhdC53aWtpAhYyMDE5MDEwMS1kZXYtZHVtbXkta2V5AAImZWRpdG9yX2lkID0gYWFhYWFhYWFhYWFhYmt2a2FhYWFhYWFhYWkAAht0aW1lID4gMjAxOS0wMS0wOVQwMDo1Nzo1MloAAAYgnroNha1hSftChtxHGTnLEmM/pY8MeQS/jBSV0UNvXug=" -FATCAT_API_HOST="http://localhost:9411/v0" -ELASTICSEARCH_BACKEND="http://localhost:9200" -ELASTICSEARCH_RELEASE_INDEX="fatcat_release" -ELASTICSEARCH_CONTAINER_INDEX="fatcat_container" -GITLAB_CLIENT_ID="" -GITLAB_CLIENT_SECRET="" -IA_XAUTH_CLIENT_ID="" -IA_XAUTH_CLIENT_SECRET="" -SENTRY_DSN="" - -# These auth keys only for workers/importers; locally will fall back to -# FATCAT_API_AUTH_TOKEN -FATCAT_AUTH_WORKER_CROSSREF="" -FATCAT_AUTH_WORKER_ORCID="" -FATCAT_AUTH_WORKER_PUBMED="" -FATCAT_AUTH_WORKER_DATACITE="" diff --git a/python/example.env b/python/example.env new file mode 100644 index 00000000..4bf76e21 --- /dev/null +++ b/python/example.env @@ -0,0 +1,20 @@ +FLASK_SECRET_KEY="TODO-REPLACE-ME" +FATCAT_DOMAIN="dev.fatcat.wiki" +# This key used in tests +FATCAT_API_AUTH_TOKEN="AgEPZGV2LmZhdGNhdC53aWtpAhYyMDE5MDEwMS1kZXYtZHVtbXkta2V5AAImZWRpdG9yX2lkID0gYWFhYWFhYWFhYWFhYmt2a2FhYWFhYWFhYWkAAht0aW1lID4gMjAxOS0wMS0wOVQwMDo1Nzo1MloAAAYgnroNha1hSftChtxHGTnLEmM/pY8MeQS/jBSV0UNvXug=" +FATCAT_API_HOST="http://localhost:9411/v0" +ELASTICSEARCH_BACKEND="http://localhost:9200" +ELASTICSEARCH_RELEASE_INDEX="fatcat_release" +ELASTICSEARCH_CONTAINER_INDEX="fatcat_container" +GITLAB_CLIENT_ID="" +GITLAB_CLIENT_SECRET="" +IA_XAUTH_CLIENT_ID="" +IA_XAUTH_CLIENT_SECRET="" +SENTRY_DSN="" + +# These auth keys only for workers/importers; locally will fall back to +# FATCAT_API_AUTH_TOKEN +FATCAT_AUTH_WORKER_CROSSREF="" +FATCAT_AUTH_WORKER_ORCID="" +FATCAT_AUTH_WORKER_PUBMED="" +FATCAT_AUTH_WORKER_DATACITE="" diff --git a/python/tests/fixtures.py b/python/tests/fixtures.py index 3cc275b3..d7ebdac8 100644 --- a/python/tests/fixtures.py +++ b/python/tests/fixtures.py @@ -11,7 +11,7 @@ import fatcat_client @pytest.fixture def full_app(): - load_dotenv(dotenv_path="./env.example") + load_dotenv(dotenv_path="./example.env") fatcat_web.app.testing = True fatcat_web.app.debug = False return fatcat_web.app @@ -22,7 +22,7 @@ def app(full_app): @pytest.fixture def api(): - load_dotenv(dotenv_path="./env.example") + load_dotenv(dotenv_path="./example.env") conf = fatcat_client.Configuration() conf.host = "http://localhost:9411/v0" conf.api_key["Authorization"] = os.getenv("FATCAT_API_AUTH_TOKEN") diff --git a/rust/INSTALL.md b/rust/INSTALL.md index 7b7d6001..462072dc 100644 --- a/rust/INSTALL.md +++ b/rust/INSTALL.md @@ -2,12 +2,12 @@ Canonical IA production/QA ansible scripts are in the journal-infra repo. These directions are likely to end up out-of-date. -## Simple Deployment +## Simple Server Deployment To install manually, on a bare server, as root: adduser fatcat - apt install postgresql-9.6 postgresql-contrib postgresql-client-9.6 \ + apt install postgresql-11 postgresql-contrib postgresql-client-11 \ nginx build-essential git pkg-config libssl-dev libpq-dev \ htop screen mkdir -p /srv/fatcat @@ -16,7 +16,6 @@ To install manually, on a bare server, as root: # setup new postgres user su - postgres createuser -P -s fatcat # strong random password - # DELETE: createdb fatcat # as fatcat user su - fatcat @@ -25,7 +24,7 @@ To install manually, on a bare server, as root: source $HOME/.cargo/env cargo install diesel_cli --no-default-features --features "postgres" cd /srv/fatcat - git clone git@git.archive.org:webgroup/fatcat + git clone https://github.com/internetarchive/fatcat.git cd rust cargo build echo "DATABASE_URL=postgres://fatcat@localhost/fatcat" > .env diff --git a/rust/README.md b/rust/README.md index 7393322d..e191d03e 100644 --- a/rust/README.md +++ b/rust/README.md @@ -21,26 +21,36 @@ do development work: - rust stable, 2018 edition, 1.32+ (eg, via "rustup", includes cargo tool) - diesel (`cargo install diesel_cli`) - postgres (compatible with 9.6+; run 11.x in production) -- postgres libs (debian: `sudo apt install libsqlite3-dev libpq-dev`) -- libsodium library and development headers (debian: `libsodium-dev`) +- postgres libs (debian/ubuntu: `libsqlite3-dev libpq-dev`) +- libsodium library and development headers (debian/ubuntu: `libsodium-dev`) -Copying commands out of `../.gitlab-ci.yml` file may be the fastest way to get -started. +We need to create a new `fatcat` postgres user and database to run tests and +develop with. On debian/ubuntu, a UNIX account named `postgres` is +automatically created with control over the database, so we'll run setup +commands from that user. To create the database account: -Create a new postgres superuser. A regular postgres user and an existing -database should also work (with up/down migrations), but it's easier to just -blow the entire database away. + sudo su - postgres -Copy `env.example` to `.env`, update if needed, then re-create database from -scratch: + # this command creates a PostgreSQL user, not a UNIX/system user + createuser -s fatcat -P + + # switch back to your regular system user + exit + +Copy `./example.env` to `./.env` and update the `DATABASE_URL` and +`TEST_DATABASE_URL` lines with the password you set above. + +As your regular user, use `diesel` to create and initialize the `fatcat` +database (`diesel` and the fatcat tools will automatically use postgresql +credentials from the `.env` file): diesel database reset -Build and run: +Build and run the API server: cargo run --bin fatcatd -Tests: +Run tests: cargo test -- --test-threads 1 diff --git a/rust/env.example b/rust/env.example deleted file mode 100644 index bee653c2..00000000 --- a/rust/env.example +++ /dev/null @@ -1,10 +0,0 @@ -DATABASE_URL="postgres://fatcat:tactaf@localhost/fatcat" -TEST_DATABASE_URL="postgres://fatcat:tactaf@localhost/fatcat_test" -AUTH_LOCATION="dev.fatcat.wiki" -AUTH_KEY_IDENT="20190101-dev-dummy-key" -AUTH_SECRET_KEY="5555555555555555555555555555555555555555xms=" -AUTH_ALT_KEYS="20181220-dev:6666666666666666666666666666666666666666xms=,20181210-dev:7777777777777777777777777777777777777777xms=" -#SENTRY_DSN= -# should be localhost IP, not 'localhost' -#FATCAT_STATSD_HOST="127.0.0.1" -#FATCAT_STATSD_PORT="8125" diff --git a/rust/example.env b/rust/example.env new file mode 100644 index 00000000..bee653c2 --- /dev/null +++ b/rust/example.env @@ -0,0 +1,10 @@ +DATABASE_URL="postgres://fatcat:tactaf@localhost/fatcat" +TEST_DATABASE_URL="postgres://fatcat:tactaf@localhost/fatcat_test" +AUTH_LOCATION="dev.fatcat.wiki" +AUTH_KEY_IDENT="20190101-dev-dummy-key" +AUTH_SECRET_KEY="5555555555555555555555555555555555555555xms=" +AUTH_ALT_KEYS="20181220-dev:6666666666666666666666666666666666666666xms=,20181210-dev:7777777777777777777777777777777777777777xms=" +#SENTRY_DSN= +# should be localhost IP, not 'localhost' +#FATCAT_STATSD_HOST="127.0.0.1" +#FATCAT_STATSD_PORT="8125" -- cgit v1.2.3