diff options
| -rw-r--r-- | extra/sql_dumps/README.md | 19 | ||||
| -rwxr-xr-x | extra/sql_dumps/ident_table_snapshot.sh | 3 | ||||
| -rw-r--r-- | rust/README.export.md | 8 |
3 files changed, 19 insertions, 11 deletions
diff --git a/extra/sql_dumps/README.md b/extra/sql_dumps/README.md index 45adae09..4d420190 100644 --- a/extra/sql_dumps/README.md +++ b/extra/sql_dumps/README.md @@ -29,14 +29,14 @@ forwarding anyways. In production: - sudo -u postgres psql fatcat_prod < dump_abstracts.sql | egrep -v ^BEGIN$ | egrep -v ^ROLLBACK$ | pv -l | gzip > /srv/fatcat/snapshots/abstracts.json.gz - sudo -u postgres psql fatcat_prod < dump_file_hashes.sql | egrep -v ^BEGIN$ | egrep -v ^ROLLBACK$ | pv -l | gzip > /srv/fatcat/snapshots/file_hashes.tsv.gz - sudo -u postgres psql fatcat_prod < dump_release_extid.sql | egrep -v ^BEGIN$ | egrep -v ^ROLLBACK$ | pv -l | gzip > /srv/fatcat/snapshots/release_extid.tsv.gz + sudo -u postgres psql fatcat_prod < dump_abstracts.sql | egrep -v ^BEGIN$ | egrep -v ^ROLLBACK$ | pv -l | pigz > /srv/fatcat/snapshots/abstracts.json.gz + sudo -u postgres psql fatcat_prod < dump_file_hashes.sql | egrep -v ^BEGIN$ | egrep -v ^ROLLBACK$ | pv -l | pigz > /srv/fatcat/snapshots/file_hashes.tsv.gz + sudo -u postgres psql fatcat_prod < dump_release_extid.sql | egrep -v ^BEGIN$ | egrep -v ^ROLLBACK$ | pv -l | pigz > /srv/fatcat/snapshots/release_extid.tsv.gz ## HOWTO: Full private database backup and restore export DATESLUG="`date +%Y-%m-%d.%H%M%S`" - time sudo -u postgres pg_dump --verbose --format=tar fatcat_prod | gzip > /srv/fatcat/snapshots/fatcat_private_dbdump_${DATESLUG}.tar.gz + time sudo -u postgres pg_dump --verbose --format=tar fatcat_prod | pigz > /srv/fatcat/snapshots/fatcat_private_dbdump_${DATESLUG}.tar.gz NOTE: by using the "directory" export (along with `--file`) instead of "tar" export, it would be possible to use parallel dumping. However, this would put @@ -45,7 +45,14 @@ issues with users/permissions. To restore, CAREFULLY, run: - sudo -u postgres pg_restore --clean --if-exists --create -exit-on-error --jobs=16 DUMP_FILE.tar.gz + sudo -u postgres pg_restore --clean --if-exists --create --exit-on-error --jobs=16 DUMP_FILE.tar.gz + + zcat sudo -u postgres pg_restore --clean --if-exists --create --exit-on-error --jobs=16 DUMP_FILE.tar.gz + +Or, in production: + + sudo su postgres + time zcat fatcat_private_dbdump_2020-02-02.022209.tar.gz | pg_restore --exit-on-error --clean --if-exists --dbname fatcat_prod To just inspect a dump: @@ -59,6 +66,6 @@ replace the `fatcat_prod` database name. # TODO: for production, probably want consistent serialization mode export DATESLUG="`date +%Y-%m-%d.%H%M%S`" - sudo -u postgres pg_dump --verbose --format=tar --exclude-table-data=auth_oidc fatcat_prod | gzip > /srv/fatcat/snapshots/fatcat_public_dbdump_${DATESLUG}.tar.gz + sudo -u postgres pg_dump --verbose --format=tar --exclude-table-data=auth_oidc fatcat_prod | pigz > /srv/fatcat/snapshots/fatcat_public_dbdump_${DATESLUG}.tar.gz Can also run using the remote/SSH options above. diff --git a/extra/sql_dumps/ident_table_snapshot.sh b/extra/sql_dumps/ident_table_snapshot.sh index 16c91612..dbd4caf0 100755 --- a/extra/sql_dumps/ident_table_snapshot.sh +++ b/extra/sql_dumps/ident_table_snapshot.sh @@ -22,7 +22,8 @@ CHANGELOG_REV="`head -n1 /tmp/fatcat_ident_latest_changelog.tsv`" OUTFILE="${OUTPUT_DIR}/fatcat_idents.$DATESLUG.r$CHANGELOG_REV.tar.gz" echo "Compressing..." -tar -C /tmp -c --gzip --verbose \ +tar -C /tmp -c --verbose \ + --use-compress-program=pigz \ -f $OUTFILE \ fatcat_ident_latest_changelog.tsv \ fatcat_ident_containers.tsv \ diff --git a/rust/README.export.md b/rust/README.export.md index f02c7f45..6a550d7d 100644 --- a/rust/README.export.md +++ b/rust/README.export.md @@ -7,8 +7,8 @@ Then, to dump locally to stdout: Or, in production: - cat /tmp/fatcat_ident_releases.tsv | ./target/release/fatcat-export release --expand files,filesets,webcaptures,container -j8 | gzip > /srv/fatcat/snapshots/release_export_expanded.json.gz - cat /tmp/fatcat_ident_releases.tsv | ./target/release/fatcat-export release -j8 | gzip > /srv/fatcat/snapshots/release_export.json.gz - cat /tmp/fatcat_ident_containers.tsv | ./target/release/fatcat-export container -j8 | gzip > /srv/fatcat/snapshots/container_export.json.gz - cat /tmp/fatcat_ident_files.tsv | ./target/release/fatcat-export file -j8 | gzip > /srv/fatcat/snapshots/file_export.json.gz + cat /tmp/fatcat_ident_releases.tsv | ./target/release/fatcat-export release --expand files,filesets,webcaptures,container -j8 | pigz > /srv/fatcat/snapshots/release_export_expanded.json.gz + cat /tmp/fatcat_ident_releases.tsv | ./target/release/fatcat-export release -j8 | pigz > /srv/fatcat/snapshots/release_export.json.gz + cat /tmp/fatcat_ident_containers.tsv | ./target/release/fatcat-export container -j8 | pigz > /srv/fatcat/snapshots/container_export.json.gz + cat /tmp/fatcat_ident_files.tsv | ./target/release/fatcat-export file -j8 | pigz > /srv/fatcat/snapshots/file_export.json.gz |