aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBryan Newbold <bnewbold@robocracy.org>2018-09-11 19:04:49 -0700
committerBryan Newbold <bnewbold@robocracy.org>2018-09-11 19:04:49 -0700
commit459ca4e1aa4a22e4adf3c275a80368949dd52e8c (patch)
treea2586c1fb87723f758801919270964240ada4d78
parentf5812c8c3b062b5efb34e45702ee7df507f71e16 (diff)
downloadfatcat-459ca4e1aa4a22e4adf3c275a80368949dd52e8c.tar.gz
fatcat-459ca4e1aa4a22e4adf3c275a80368949dd52e8c.zip
first pass fast export
-rw-r--r--rust/Cargo.lock170
-rw-r--r--rust/Cargo.toml3
-rw-r--r--rust/src/bin/fatcat-export.rs161
-rw-r--r--rust/src/lib.rs2
4 files changed, 335 insertions, 1 deletions
diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index 5933716d..6d12659e 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -20,6 +20,14 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
+name = "arrayvec"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
name = "ascii"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -158,6 +166,14 @@ dependencies = [
]
[[package]]
+name = "cloudabi"
+version = "0.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "bitflags 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
name = "conduit-mime-types"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -166,6 +182,36 @@ dependencies = [
]
[[package]]
+name = "crossbeam-channel"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "crossbeam-epoch 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)",
+ "cfg-if 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
name = "data-encoding"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -305,6 +351,7 @@ version = "0.1.0"
dependencies = [
"chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-channel 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"data-encoding 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"diesel 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"diesel_migrations 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -529,6 +576,15 @@ version = "0.2.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
+name = "lock_api"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
name = "log"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -566,6 +622,11 @@ dependencies = [
]
[[package]]
+name = "memoffset"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
name = "migrations_internals"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -627,6 +688,11 @@ dependencies = [
]
[[package]]
+name = "nodrop"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
name = "num"
version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -690,6 +756,35 @@ dependencies = [
]
[[package]]
+name = "owning_ref"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "lock_api 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
name = "percent-encoding"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -811,6 +906,23 @@ dependencies = [
]
[[package]]
+name = "rand"
+version = "0.5.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)",
+ "rand_core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
+ "winapi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
name = "redox_syscall"
version = "0.1.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -897,6 +1009,14 @@ version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
+name = "rustc_version"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
name = "safemem"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -910,6 +1030,24 @@ dependencies = [
]
[[package]]
+name = "scopeguard"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
+name = "semver"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "semver-parser"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
name = "serde"
version = "1.0.55"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -980,6 +1118,19 @@ dependencies = [
]
[[package]]
+name = "smallvec"
+version = "0.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+
+[[package]]
name = "strsim"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1288,6 +1439,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum aho-corasick 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d6531d44de723825aa81398a6415283229725a00fa30713812ab9323faa82fc4"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
"checksum antidote 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "34fde25430d87a9388dadbe6e34d7f72a462c8b43ac8d309b42b0a8505d7e2a5"
+"checksum arrayvec 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "a1e964f9e24d588183fcb43503abda40d288c8657dfc27311516ce2f05675aef"
"checksum ascii 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ae7d751998c189c1d4468cf0a39bb2eae052a9c58d50ebb3b9591ee3813ad50"
"checksum atty 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "2fc4a1aa4c24c0718a250f0681885c1af91419d242f29eb8f2ab28502d80dbd1"
"checksum backtrace 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "8ea58cd16fd6c9d120b5bcb01d63883ae4cc7ba2aed35c1841b862a3c7ef6639"
@@ -1305,7 +1457,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum chrono 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1cce36c92cb605414e9b824f866f5babe0a0368e39ea07393b9b63cf3844c0e6"
"checksum chunked_transfer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "498d20a7aaf62625b9bf26e637cf7736417cde1d0c99f1d04d1170229a85cf87"
"checksum clap 2.31.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f16b89cbb9ee36d87483dc939fe9f1e13c05898d56d7b230a0d4dff033a536"
+"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
"checksum conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "95ca30253581af809925ef68c2641cc140d6183f43e12e0af4992d53768bd7b8"
+"checksum crossbeam-channel 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a5716fadb87a5633db34c5e83ee6e036e6edc229f8a6bfb7c7c84ed340ba95df"
+"checksum crossbeam-epoch 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9c90f1474584f38e270b5b613e898c8c328aa4f3dea85e0a27ac2e642f009416"
+"checksum crossbeam-utils 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "677d453a17e8bd2b913fa38e8b9cf04bcdbb5be790aa294f2389661d72036015"
"checksum data-encoding 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "67df0571a74bf0d97fb8b2ed22abdd9a48475c96bd327db968b7d9cace99655e"
"checksum diesel 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e71e7a348ae6064e86c4cf0709f0e4c3ef6f30e8e7d3dc05737164af4ebd3511"
"checksum diesel_derives 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "03bcaf77491f53e400d5ee3bdd57142ea4e1c47fe9217b3361ff9a76ca0e3d37"
@@ -1342,17 +1498,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73"
"checksum lazy_static 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c8f31047daa365f19be14b47c29df4f7c3b581832407daabe6ae77397619237d"
"checksum libc 0.2.40 (registry+https://github.com/rust-lang/crates.io-index)" = "6fd41f331ac7c5b8ac259b8bf82c75c0fb2e469bbf37d2becbba9a6a2221965b"
+"checksum lock_api 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "949826a5ccf18c1b3a7c3d57692778d21768b79e46eb9dd07bfc4c2160036c54"
"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b"
"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2"
"checksum matches 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "100aabe6b8ff4e4a7e32c1c13523379802df0772b82466207ac25b013f193376"
"checksum memchr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "148fab2e51b4f1cfc66da2a7c32981d1d3c083a803978268bb11fe4b86925e7a"
"checksum memchr 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "796fba70e76612589ed2ce7f45282f5af869e0fdd7cc6199fa1aa1f1d591ba9d"
+"checksum memoffset 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0f9dc261e2b62d7a622bf416ea3c5245cdd5d9a7fcc428c0d06804dfce1775b3"
"checksum migrations_internals 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8cf7c8c4f83fa9f47440c0b4af99973502de55e6e7b875f693bd263e03f93e7e"
"checksum migrations_macros 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "79f12499ef7353bdeca2d081bc61edd8351dac09a33af845952009b5a3d68c1a"
"checksum mime 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ba626b8a6de5da682e1caa06bdb42a335aee5a84db8e5046a3e8ab17ba0a3ae0"
"checksum mime_guess 1.8.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b7e2b09d08313f84e0fb82d13a4d859109a17543fe9af3b6d941dc1431f7de79"
"checksum modifier 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "41f5c9112cb662acd3b204077e0de5bc66305fa8df65c8019d5adb10e9ab6e58"
"checksum multipart 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)" = "92f54eb45230c3aa20864ccf0c277eeaeadcf5e437e91731db498dbf7fbe0ec6"
+"checksum nodrop 0.1.12 (registry+https://github.com/rust-lang/crates.io-index)" = "9a2228dca57108069a5262f2ed8bd2e82496d2e074a06d1ccc7ce1687b6ae0a2"
"checksum num 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "4703ad64153382334aa8db57c637364c322d3372e097840c72000dabdcf6156e"
"checksum num-integer 0.1.38 (registry+https://github.com/rust-lang/crates.io-index)" = "6ac0ea58d64a89d9d6b7688031b3be9358d6c919badcf7fbb0527ccfd891ee45"
"checksum num-iter 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "af3fdbbc3291a5464dc57b03860ec37ca6bf915ed6ee385e7c6c052c422b2124"
@@ -1360,6 +1519,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30"
"checksum openssl 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)" = "a3605c298474a3aa69de92d21139fb5e2a81688d308262359d85cdd0d12a7985"
"checksum openssl-sys 0.9.30 (registry+https://github.com/rust-lang/crates.io-index)" = "73ae718c3562989cd3a0a5c26610feca02f8116822f6f195e6cf4887481e57f5"
+"checksum owning_ref 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cdf84f41639e037b484f93433aa3897863b561ed65c6e59c7073d7c561710f37"
+"checksum parking_lot 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f0802bff09003b291ba756dc7e79313e51cc31667e94afbe847def490424cde5"
+"checksum parking_lot_core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ad7f7e6ebdc79edff6fdcb87a55b620174f7a989e3eb31b65231f4af57f00b8c"
"checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831"
"checksum persistent 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8e8fa0009c4f3d350281309909c618abddf10bb7e3145f28410782f6a5ec74c5"
"checksum phf 0.7.22 (registry+https://github.com/rust-lang/crates.io-index)" = "7d37a244c75a9748e049225155f56dbcb98fe71b192fd25fd23cb914b5ad62f2"
@@ -1375,6 +1537,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum r2d2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f9078ca6a8a5568ed142083bb2f7dc9295b69d16f867ddcc9849e51b17d8db46"
"checksum rand 0.3.22 (registry+https://github.com/rust-lang/crates.io-index)" = "15a732abf9d20f0ad8eeb6f909bf6868722d9a06e1e50802b6a70351f40b4eb1"
"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"
+"checksum rand 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e464cd887e869cddcae8792a4ee31d23c7edd516700695608f5b98c67ee0131c"
+"checksum rand_core 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "edecf0f94da5551fc9b492093e30b041a891657db7940ee221f9d2f66e82eef2"
"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd"
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
"checksum regex 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "9329abc99e39129fcceabd24cf5d85b4671ef7c29c50e972bc5afe32438ec384"
@@ -1386,8 +1550,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum router 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dc63b6f3b8895b0d04e816b2b1aa58fdba2d5acca3cbb8f0ab8e017347d57397"
"checksum rustc-demangle 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "76d7ba1feafada44f2d38eed812bd2489a03c0f5abb975799251518b68848649"
"checksum rustc-serialize 0.3.24 (registry+https://github.com/rust-lang/crates.io-index)" = "dcf128d1287d2ea9d80910b5f1120d0b8eede3fbf1abe91c40d39ea7d51e6fda"
+"checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f"
"checksum scheduled-thread-pool 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1a2ff3fc5223829be817806c6441279c676e454cc7da608faf03b0ccc09d3889"
+"checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27"
+"checksum semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
+"checksum semver-parser 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
"checksum serde 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)" = "97f6a6c3caba0cf8f883b53331791036404ce3c1bd895961cf8bb2f8cecfd84b"
"checksum serde_derive 1.0.55 (registry+https://github.com/rust-lang/crates.io-index)" = "f51b0ef935cf8a41a77bce553da1f8751a739b7ad82dd73669475a22e6ecedb0"
"checksum serde_ignored 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "190e9765dcedb56be63b6e0993a006c7e3b071a016a304736e4a315dc01fb142"
@@ -1397,6 +1565,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum slog 2.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2f7bfce6405155042d42ec0e645efe43eddedd7be280063ce0623b120014e7f9"
"checksum slog-async 2.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e544d16c6b230d84c866662fe55e31aacfca6ae71e6fc49ae9a311cb379bfc2f"
"checksum slog-term 2.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5951a808c40f419922ee014c15b6ae1cd34d963538b57d8a4778b9ca3fff1e0b"
+"checksum smallvec 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "153ffa32fd170e9944f7e0838edf824a754ec4c1fc64746fcc9fe1f8fa602e5d"
+"checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8"
"checksum strsim 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb4f380125926a99e52bc279241539c018323fab05ad6368b56f93d9369ff550"
"checksum swagger 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "190ef0c6327759d0beb76d969b236fa3cc42469f9e107f626bbcc152727b4d12"
"checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad"
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
index 6b2e1ffe..0e14c583 100644
--- a/rust/Cargo.toml
+++ b/rust/Cargo.toml
@@ -35,6 +35,9 @@ slog-term = "*"
slog-async = "*"
serde_json = "1.0"
+# Command-line tools
+crossbeam-channel = "0.2"
+
# Unused (hyper 0.11 and https)
#hyper-openssl = {version = "0.2", optional = true}
#hyper-tls = {version = "0.1.2", optional = true}
diff --git a/rust/src/bin/fatcat-export.rs b/rust/src/bin/fatcat-export.rs
new file mode 100644
index 00000000..d7371c87
--- /dev/null
+++ b/rust/src/bin/fatcat-export.rs
@@ -0,0 +1,161 @@
+//! JSON Export Helper
+
+extern crate clap;
+extern crate diesel;
+extern crate dotenv;
+#[macro_use]
+extern crate error_chain;
+extern crate fatcat;
+extern crate fatcat_api_spec;
+#[macro_use]
+extern crate slog;
+extern crate slog_async;
+extern crate slog_term;
+extern crate uuid;
+extern crate crossbeam_channel;
+extern crate serde_json;
+
+use clap::{App, Arg};
+use slog::{Drain, Logger};
+use dotenv::dotenv;
+use std::env;
+
+use diesel::prelude::*;
+use diesel::r2d2::ConnectionManager;
+use fatcat_api_spec::models::*;
+use std::str::FromStr;
+use uuid::Uuid;
+use fatcat::ConnectionPool;
+use fatcat::api_helpers::*;
+use fatcat::api_entity_crud::*;
+use fatcat::errors::*;
+
+use std::thread;
+//use std::io::{Stdout,StdoutLock};
+use crossbeam_channel as channel;
+//use num_cpus; TODO:
+use std::io::prelude::*;
+use std::io::{BufReader, BufWriter};
+
+const CHANNEL_BUFFER_LEN: usize = 200;
+
+struct IdentRow {
+ ident_id: FatCatId,
+ rev_id: Option<Uuid>,
+ redirect_id: Option<FatCatId>,
+}
+
+/// Instantiate a new API server with a pooled database connection
+pub fn database_worker_pool() -> Result<ConnectionPool> {
+ dotenv().ok();
+ let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set");
+ let manager = ConnectionManager::<PgConnection>::new(database_url);
+ let pool = diesel::r2d2::Pool::builder()
+ .build(manager)
+ .expect("Failed to create database pool.");
+ Ok(pool)
+}
+
+fn loop_work(row_receiver: channel::Receiver<IdentRow>, output_sender: channel::Sender<String>, db_conn: &DbConn, expand: Option<ExpandFlags>) -> Result<()> {
+ for row in row_receiver {
+ // TODO: based on types
+ let mut entity = ReleaseEntity::db_get_rev(db_conn, row.rev_id.expect("valid, non-deleted row"))?;
+ entity.state = Some("active".to_string()); // TODO
+ entity.ident = Some(row.ident_id.to_string());
+ if let Some(expand) = expand {
+ entity.db_expand(db_conn, expand)?;
+ }
+ output_sender.send(serde_json::to_string(&entity)?);
+ }
+ Ok(())
+}
+
+fn loop_printer(output_receiver: channel::Receiver<String>, done_sender: channel::Sender<()>) -> Result<()> {
+ let output = std::io::stdout();
+ // XXX should log...
+ // let mut buf_output = BufWriter::new(output.lock());
+ let mut buf_output = BufWriter::new(output);
+ for line in output_receiver {
+ buf_output.write_all(&line.into_bytes())?;
+ buf_output.write(b"\n")?;
+ buf_output.flush()?;
+ }
+ drop(done_sender);
+ Ok(())
+}
+
+fn parse_line(s: String) -> Result<IdentRow> {
+ let fields: Vec<String> = s.split("\t").map(|v| v.to_string()).collect();
+ if fields.len() != 3 {
+ bail!("Invalid input line");
+ }
+ Ok(IdentRow {
+ ident_id: FatCatId::from_uuid(&Uuid::from_str(&fields[0])?),
+ rev_id: Some(Uuid::from_str(&fields[1])?),
+ redirect_id: None,
+ })
+}
+
+#[test]
+fn test_parse_line() {
+ assert!(false)
+}
+
+// Use num_cpus/2, or CLI arg for worker count
+//
+// 1. open buffered reader, buffered writer, and database pool. create channels
+// 2. spawn workers:
+// - get a database connection from database pool
+// - iterate over row channel, pushing Strings to output channel
+// - exit when end of rows
+// 3. spawn output printer
+// 4. read rows, pushing to row channel
+// => every N lines, log to stderr
+// 5. wait for all channels to finish, and quit
+pub fn run() -> Result<()> {
+
+ let db_pool = database_worker_pool()?;
+ let buf_input = BufReader::new(std::io::stdin());
+ let worker_count = 4;
+ let (row_sender, row_receiver) = channel::bounded(CHANNEL_BUFFER_LEN);
+ let (output_sender, output_receiver) = channel::bounded(CHANNEL_BUFFER_LEN);
+ let (done_sender, done_receiver) = channel::bounded(0);
+
+ // Start row worker threads
+ for _ in 0..worker_count {
+ let db_conn = db_pool.get().expect("database pool");
+ let row_receiver = row_receiver.clone();
+ let output_sender = output_sender.clone();
+ thread::spawn(move || loop_work(row_receiver, output_sender, &db_conn, None));
+ }
+ drop(output_sender);
+ // Start printer thread
+ thread::spawn(move || loop_printer(output_receiver, done_sender));
+
+ for line in buf_input.lines() {
+ let line = line?;
+ let row = parse_line(line)?;
+ row_sender.send(row);
+ }
+ drop(row_sender);
+ done_receiver.recv();
+ Ok(())
+}
+
+fn main() {
+ /*
+ let matches = App::new("server")
+ .arg(
+ Arg::with_name("https")
+ .long("https")
+ .help("Whether to use HTTPS or not"),
+ )
+ .get_matches();
+
+ let decorator = slog_term::TermDecorator::new().build();
+ let drain = slog_term::CompactFormat::new(decorator).build().fuse();
+ let drain = slog_async::Async::new(drain).build().fuse();
+ let logger = Logger::root(drain, o!());
+ */
+ run().expect("success")
+}
diff --git a/rust/src/lib.rs b/rust/src/lib.rs
index 381c67ea..d3ef4a9c 100644
--- a/rust/src/lib.rs
+++ b/rust/src/lib.rs
@@ -29,7 +29,7 @@ pub mod api_wrappers;
pub mod database_models;
pub mod database_schema;
-mod errors {
+pub mod errors {
// Create the Error, ErrorKind, ResultExt, and Result types
error_chain! {
foreign_links { Fmt(::std::fmt::Error);