From 04858ed7e251227e70a7105c8197abb194b6195a Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Sun, 19 May 2019 22:06:06 -0700 Subject: postgres temp databases --- software/postgres.page | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'software') diff --git a/software/postgres.page b/software/postgres.page index bdb01d8..7d05074 100644 --- a/software/postgres.page +++ b/software/postgres.page @@ -15,3 +15,20 @@ Basic status from the psql command line: \l+ (list databases) \dt+ (describe table) + +## Temporary Databases in Tests + +Try `pg_tmp`, though it may not work with high concurrency (due to internal +postgres locking), and might need special configuration in CI environments. + +Debian also ships with `pg_virtualenv` for temporary databases. + +--------- + +More refs/links: + +- https://stackoverflow.com/questions/9604723/alternate-output-format-for-psql +- https://brandur.org/postgres-connections +- https://www.citusdata.com/blog/2017/07/16/customizing-my-postgres-shell-using-psqlrc/ +- `bind "^R" em-inc-search-prev` + -- cgit v1.2.3 From 8c51b17ef7176de9ca7afdb5f1d51dd3905ba555 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Sun, 19 May 2019 22:09:30 -0700 Subject: rust Option/Result map() tricks --- software/rust.page | 44 +++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) (limited to 'software') diff --git a/software/rust.page b/software/rust.page index df99de4..4e8aa44 100644 --- a/software/rust.page +++ b/software/rust.page @@ -3,9 +3,10 @@ Rust ## Resources -- [http://xion.io/post/code/rust-iter-patterns.html]() -- [https://deterministic.space/rust-cli-tips.html]() -- [https://manishearth.github.io/blog/2018/01/10/whats-tokio-and-async-io-all-about/]() +- +- +- +- Optimization: use `RUSTFLAGS="-C target-cpu=native"` to take advantage of CPU special features. @@ -22,3 +23,40 @@ Run tests with stdout output: To run tests with logging enabled (eg, with `env_logger`), make sure you add `env_logger::init()` to the test function itself. + + +## map() and Result Ergonomics + +`.collect()` has some magical features! In addition to turning an iterator of +`Item` into `Vec`, it will turn an iterator of `Result` into +`Result>`. This makes it really useful for the end of functions. + +This is particularly useful for resolving some categories of "error handling in +map closures": you can use `?` in the map closure as long as you wrap the happy +path with `Ok()` and call collect on the outside. Eg: + + let list: Vec = junk + .iter() + .map(|thing| Ok(Item { + a: thing.a, + b: fixup(thing.widget)?, + })) + .collect::Result>()?; + +What about when `map` over an `Option`? Eg: + + let toy = Shiny { + a: 123, + b: component.map(|v| paint(v).expect("paint to succeed"), + }; + +Should use match in this case: + + let toy = Shiny { + a: 123, + b: match component { + None => None, + Some(v) => Some(paint(v)?), + }, + }; + -- cgit v1.2.3 From 2f4bae6458b40e30b353743d21151dcf3aa346a3 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Sun, 19 May 2019 22:09:45 -0700 Subject: another bash reference --- software/bash.page | 2 ++ 1 file changed, 2 insertions(+) (limited to 'software') diff --git a/software/bash.page b/software/bash.page index 0d5325b..5d82fdc 100644 --- a/software/bash.page +++ b/software/bash.page @@ -37,6 +37,8 @@ Note that `join`, `grep`, and others sometimes exit non-zero return codes on purpose (eg, pipe input closed or found no matches, as expected), which makes life difficult. Sometimes `|| true` is enough to get around this. +More on this: + ## General Style Google has a style guide: https://google.github.io/styleguide/shell.xml -- cgit v1.2.3 From 4579dc274924ce7987c6990f69a2b735a37ed7b4 Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 29 May 2019 13:39:37 -0700 Subject: python memory debugging --- software/python.page | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) (limited to 'software') diff --git a/software/python.page b/software/python.page index 0fd2531..0fc3c8d 100644 --- a/software/python.page +++ b/software/python.page @@ -205,3 +205,24 @@ and create a `pytest.ini` like: norecursedirs = .svn _build tmp* Need to mock? + +Debugging Memory Usage +------------------------ + +Most helpful tools I found were `psutil` and `pympler` (both need to be +installed). + + import os, psutil + process = psutil.Process(os.getpid()) + print(process.memory_info().rss) + # ... do some stuff ... + print(process.memory_info().rss) + +and + + from pympler import tracker + tr = tracker.SummaryTracker() + tr.print_diff() + + # ... do some stuff ... + tr.print_diff() -- cgit v1.2.3 From e59ae83dd94ca1f468cc507f5d420def0d6983ed Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Wed, 29 May 2019 13:40:26 -0700 Subject: rsyslog notes --- software/rsyslog.page | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 software/rsyslog.page (limited to 'software') diff --git a/software/rsyslog.page b/software/rsyslog.page new file mode 100644 index 0000000..b55c127 --- /dev/null +++ b/software/rsyslog.page @@ -0,0 +1,20 @@ + +Put these in files like `/etc/rsyslog.d/40-grobid-skip.conf`. + +Filter out lines from a given program: + + :programname, isequal, "GROBID" ~ + +Or matching a pattern: + + :msg, startswith, "inserting line to HBase: sha1:" + :msg, contains, " INFO GET http" + +For list of operations ("isequal", "regex", "contains"): + + https://www.rsyslog.com/doc/v8-stable/configuration/filters.html + +For list of properties (":msg", ":programname"): + + https://www.rsyslog.com/doc/v8-stable/configuration/properties.html + -- cgit v1.2.3 From 1c5f02be0189e905965395c2083ee5dab5fb71ff Mon Sep 17 00:00:00 2001 From: Bryan Newbold Date: Mon, 18 May 2020 16:08:16 -0700 Subject: rust learning notes --- software/rust.page | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) (limited to 'software') diff --git a/software/rust.page b/software/rust.page index 4e8aa44..dc900c8 100644 --- a/software/rust.page +++ b/software/rust.page @@ -60,3 +60,76 @@ Should use match in this case: }, }; +## 2020-05-17 Reading + +While working on fatcat-cli tool, checked the The Rust Programming Language +book to read about trait objects and the `dyn` keyword, which I had ignored +previously. + +They seem like they could be used in a few places in fatcat-server rust code. +We don't particularly care about per-function-call performance there, and most +entities are already allocated on the heap. + +Other small syntax and thing learned: + +Can copy a struct while only updating specific fields with ".." syntax. Might +use this in fatcat-cli for update mutation. + +This is the cleanest example of using ErrorKind that I have seen: + + let f = match f { + Ok(file) => file, + Err(error) => match error.kind() { + ErrorKind::NotFound => match File::create("hello.txt") { + Ok(fc) => fc, + Err(e) => panic!("Problem creating the file: {:?}", e), + }, + other_error => { + panic!("Problem opening the file: {:?}", other_error) + } + }, + }; + +I didn't realize that test code may get compiled into non-test binaries unless +annotated with `#[cfg(test)]`. You are supposed to create a sub-module within +each `src/` file with unittests, like: + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } + } +This doesn't apply to `tests/` directory, which is for integration tests. + +The common pattern for binary crates (vs. library crates) is to have `main.rs` +and `lib.rs`, with any code that needs to be tested in `lib.rs` (aka, all the +actual logic). + +I think I knew `eprintln!()` (for stderr) vs. `println!()` (for stdout), but +good to remember. + +There is a description of how to avoid memory leaks with reference counting +using "weak" `Rc` references. Probably worth reading the [entire chapter on smart pointers](https://doc.rust-lang.org/book/ch15-06-reference-cycles.html#preventing-reference-cycles-turning-an-rct-into-a-weakt) +(including Box, Rc, RefCell) again. + +For the `Sized` trait, and `Sized` trait alone, can specify an ambiguous trait +constraint with `?` to indicate "may or may not be Sized", which doesn't really +mean anything but does explicitly allow generic functions over non-sized traits +like: + + fn my_generic_func(t: &T) { + // --snip-- + } + +A trait can depend on another trait. For example, a PrettyPrint trait could +rely on Display (and impl functions could call functions from Display). This is +done on the trait definition line. Such a trait is called a "supertrait". + +Implementing Deref on a wrapper type allows transparent access to all the trait +methods on the interior object. + +Also, a new longer post on error handling: -- cgit v1.2.3