aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2018-05-10 12:11:19 -0700
committerBryan Newbold <bnewbold@robocracy.org>2018-05-10 12:11:19 -0700
commit19f9b9c1ea8919daf85e749a0cc35375587bcc15 (patch)
tree85cf7cf8cafade0ba2551ef2373755591fd7da92
parentb9200623b0be3c8b663c1000b63c3ab581b7d34b (diff)
downloadfatcat-19f9b9c1ea8919daf85e749a0cc35375587bcc15.tar.gz
fatcat-19f9b9c1ea8919daf85e749a0cc35375587bcc15.zip
dummy data and postgres notes
-rw-r--r--golang/cmd/fatcatd/reinit.go15
-rw-r--r--golang/sql/dummy-data.sql42
-rw-r--r--notes/postgres_tuning.txt8
3 files changed, 62 insertions, 3 deletions
diff --git a/golang/cmd/fatcatd/reinit.go b/golang/cmd/fatcatd/reinit.go
index b7cfa468..60726df0 100644
--- a/golang/cmd/fatcatd/reinit.go
+++ b/golang/cmd/fatcatd/reinit.go
@@ -22,12 +22,12 @@ func main_reinit() {
box := packr.NewBox("../../sql")
sql_schema, err := box.MustString("fatcat-schema.sql")
if err != nil {
- log.Panicf("finding SQL file: {}", err)
+ log.Panicf("finding SQL file: %v", err)
}
db_options, err := pg.ParseURL(viper.GetString("db_url"))
if err != nil {
- log.Panicf("parsing DB string: {}", err)
+ log.Panicf("parsing DB string: %v", err)
}
db := pg.Connect(db_options)
defer db.Close()
@@ -35,7 +35,16 @@ func main_reinit() {
log.Info("Starting load...")
_, err = db.Exec(sql_schema)
if err != nil {
- log.Fatalf("Error loading SQL: {}", err)
+ log.Fatalf("Error loading SQL: %v", err)
+ }
+ log.Info("Loading dummy data...")
+ sql_dummy, err := box.MustString("dummy-data.sql")
+ if err != nil {
+ log.Panicf("finding SQL file: %v", err)
+ }
+ _, err = db.Exec(sql_dummy)
+ if err != nil {
+ log.Fatalf("Error loading SQL: %v", err)
}
log.Info("Success!")
diff --git a/golang/sql/dummy-data.sql b/golang/sql/dummy-data.sql
new file mode 100644
index 00000000..9e01fae8
--- /dev/null
+++ b/golang/sql/dummy-data.sql
@@ -0,0 +1,42 @@
+
+-- Fake data at the raw SQL level, for early development and testing
+
+BEGIN;
+
+INSERT INTO editor (id, username, is_admin) VALUES
+ (1, 'admin', true),
+ (2, 'claire', true),
+ (3, 'doug', false);
+
+INSERT INTO editgroup (id, editor_id, description) VALUES
+ (1, 1, 'first edit ever!'),
+ (2, 1, 'another one!'),
+ (3, 3, 'user edit'),
+ (4, 2, 'uncommited edit');
+
+INSERT INTO editor (id, username, is_admin, active_editgroup_id) VALUES
+ (4, 'bnewbold', true, 4);
+
+INSERT INTO changelog (id, editgroup_id) VALUES
+ (1, 1),
+ (2, 2),
+ (3, 3);
+
+INSERT INTO creator_rev (id, name, orcid) VALUES
+ (1, 'Grace Hopper', null),
+ (2, 'Emily Noethe', null),
+ (3, 'Christine Moran', '0000-0003-2088-7465');
+
+INSERT INTO creator_ident (id, is_live, rev_id, redirect_id) VALUES
+ ('f1f046a3-45c9-4b99-adce-000000000001', true, 1, null),
+ ('f1f046a3-45c9-4b99-adce-000000000002', true, 2, null),
+ ('f1f046a3-45c9-4b99-adce-000000000003', true, 3, null),
+ ('f1f046a3-45c9-4b99-adce-000000000004', false, 2, null);
+
+INSERT INTO creator_edit (id, ident_id, rev_id, redirect_id, editgroup_id) VALUES
+ (1, 'f1f046a3-45c9-4b99-adce-000000000001', 1, null, 1),
+ (2, 'f1f046a3-45c9-4b99-adce-000000000002', 2, null, 2),
+ (3, 'f1f046a3-45c9-4b99-adce-000000000003', 3, null, 3),
+ (4, 'f1f046a3-45c9-4b99-adce-000000000004', 2, null, 4);
+
+COMMIT;
diff --git a/notes/postgres_tuning.txt b/notes/postgres_tuning.txt
new file mode 100644
index 00000000..d1e353da
--- /dev/null
+++ b/notes/postgres_tuning.txt
@@ -0,0 +1,8 @@
+
+For bulk inserts:
+
+- make write-ahead-log larger (eg, 16MB)
+- transactions of ~1000+ inserts
+- https://www.postgresql.org/docs/current/static/populate.html
+- https://www.depesz.com/2007/07/05/how-to-insert-data-to-database-as-fast-as-possible/
+- https://stackoverflow.com/questions/12206600/how-to-speed-up-insertion-performance-in-postgresql