Hi, we are currently building dev-db/sqlite with support for pread. Please consider adding support for pread{,64}. To test I used "db_bench_sqlite3" from dev-libs/leveldb: # cd /usr/portage/dev-libs/leveldb/ # ebuild leveldb-1.18-r1.ebuild clean configure # cd /var/tmp/portage/dev-libs/leveldb-1.18-r1/work/leveldb-1.18 # make db_bench_sqlite3 # cp db_bench_sqlite3 /tmp With dev-db/sqlite-3.8.9 (without pread): HDD: ==== $ ./db_bench_sqlite3 SQLite: version 3.8.9 Date: Fri May 8 19:37:01 2015 CPU: 4 * Intel(R) Core(TM) i7-3770K CPU @ 3.50GHz CPUCache: 8192 KB Keys: 16 bytes each Values: 100 bytes each Entries: 1000000 RawSize: 110.6 MB (estimated) ------------------------------------------------ fillseq : 10.109 micros/op; 10.9 MB/s fillseqsync : 153.433 micros/op; 0.7 MB/s (10000 ops) fillseqbatch : 3.382 micros/op; 32.7 MB/s fillrandom : 21.185 micros/op; 5.2 MB/s fillrandsync : 157.832 micros/op; 0.7 MB/s (10000 ops) fillrandbatch : 11.005 micros/op; 10.1 MB/s overwrite : 21.390 micros/op; 5.2 MB/s overwritebatch : 15.715 micros/op; 7.0 MB/s readrandom : 5.645 micros/op; readseq : 1.947 micros/op; 49.0 MB/s fillrand100K : 378.080 micros/op; 252.3 MB/s (1000 ops) fillseq100K : 468.469 micros/op; 203.6 MB/s (1000 ops) readseq : 44.409 micros/op; 2147.5 MB/s readrand100K : 51.067 micros/op; RAM DISK: ========= $ ./db_bench_sqlite3 SQLite: version 3.8.9 Date: Fri May 8 19:48:14 2015 CPU: 4 * Intel(R) Core(TM) i7-3770K CPU @ 3.50GHz CPUCache: 8192 KB Keys: 16 bytes each Values: 100 bytes each Entries: 1000000 RawSize: 110.6 MB (estimated) ------------------------------------------------ fillseq : 7.639 micros/op; 14.5 MB/s fillseqsync : 7.749 micros/op; 14.3 MB/s (10000 ops) fillseqbatch : 3.074 micros/op; 36.0 MB/s fillrandom : 16.689 micros/op; 6.6 MB/s fillrandsync : 9.526 micros/op; 11.6 MB/s (10000 ops) fillrandbatch : 9.227 micros/op; 12.0 MB/s overwrite : 17.695 micros/op; 6.3 MB/s overwritebatch : 13.470 micros/op; 8.2 MB/s readrandom : 6.012 micros/op; readseq : 1.996 micros/op; 47.8 MB/s fillrand100K : 242.195 micros/op; 393.8 MB/s (1000 ops) fillseq100K : 283.248 micros/op; 336.7 MB/s (1000 ops) readseq : 47.220 micros/op; 2019.6 MB/s readrand100K : 54.109 micros/op; db_bench_sqlite3 uses a hard coded path. For ramdisk test you have to do something like # mkdir /tmp/leveldbtest-0 # mount -t tmpfs -o size=1000M none /tmp/leveldbtest-0/ (yes, you need ~1gig) Now I applied to following changes to "/usr/portage/dev-db/sqlite/sqlite-3.8.9.ebuild": --- sqlite-3.8.9.ebuild.old 2015-05-10 16:56:38.000000000 +0200 +++ sqlite-3.8.9.ebuild 2015-05-10 16:57:34.000000000 +0200 @@ -106,6 +106,9 @@ # http://sqlite.org/unlock_notify.html append-cppflags -DSQLITE_ENABLE_UNLOCK_NOTIFY + # Use pread/pwrite + append-cppflags -DUSE_PREAD64 + if use icu; then append-cppflags -DSQLITE_ENABLE_ICU if amalgamation; then ...and re-run my benchmarks: RAM DISK: ========= SQLite: version 3.8.9 Date: Fri May 8 19:54:59 2015 CPU: 4 * Intel(R) Core(TM) i7-3770K CPU @ 3.50GHz CPUCache: 8192 KB Keys: 16 bytes each Values: 100 bytes each Entries: 1000000 RawSize: 110.6 MB (estimated) ------------------------------------------------ fillseq : 7.625 micros/op; 14.5 MB/s fillseqsync : 7.501 micros/op; 14.7 MB/s (10000 ops) fillseqbatch : 3.021 micros/op; 36.6 MB/s fillrandom : 16.272 micros/op; 6.8 MB/s fillrandsync : 9.206 micros/op; 12.0 MB/s (10000 ops) fillrandbatch : 8.854 micros/op; 12.5 MB/s overwrite : 17.091 micros/op; 6.5 MB/s overwritebatch : 12.899 micros/op; 8.6 MB/s readrandom : 5.674 micros/op; readseq : 1.947 micros/op; 49.0 MB/s fillrand100K : 228.538 micros/op; 417.4 MB/s (1000 ops) fillseq100K : 224.698 micros/op; 424.5 MB/s (1000 ops) readseq : 42.136 micros/op; 2263.3 MB/s readrand100K : 48.376 micros/op; HDD: ==== SQLite: version 3.8.9 Date: Fri May 8 19:57:29 2015 CPU: 4 * Intel(R) Core(TM) i7-3770K CPU @ 3.50GHz CPUCache: 8192 KB Keys: 16 bytes each Values: 100 bytes each Entries: 1000000 RawSize: 110.6 MB (estimated) ------------------------------------------------ fillseq : 9.636 micros/op; 11.5 MB/s fillseqsync : 156.583 micros/op; 0.7 MB/s (10000 ops) fillseqbatch : 3.213 micros/op; 34.4 MB/s fillrandom : 19.531 micros/op; 5.7 MB/s fillrandsync : 159.840 micros/op; 0.7 MB/s (10000 ops) fillrandbatch : 10.240 micros/op; 10.8 MB/s overwrite : 20.228 micros/op; 5.5 MB/s overwritebatch : 14.699 micros/op; 7.5 MB/s readrandom : 5.332 micros/op; readseq : 1.724 micros/op; 55.3 MB/s fillrand100K : 346.456 micros/op; 275.3 MB/s (1000 ops) fillseq100K : 356.193 micros/op; 267.8 MB/s (1000 ops) readseq : 39.081 micros/op; 2440.3 MB/s readrand100K : 45.004 micros/op; I would have proposed a patch for the ebuild but I don't know what's the best way to do that: Not sure if all our arches support pread/pwrite. On 64-bit systems you want to use pread64 instead of just pread... Reproducible: Always
Mhh... > we are currently building dev-db/sqlite with support for pread. should be obviously > we are currently building dev-db/sqlite *without* support for pread. :>
With glibc, on 64-bit architectures, pread() and pread64() probably behave identically: $ gcc -E -P - <<< "#include <unistd.h>" |& grep -E "typedef .* __off" typedef long int __off_t; typedef long int __off64_t; $ gcc -E -P -m32 - <<< "#include <unistd.h>" |& grep -E "typedef .* __off" __extension__ typedef long int __off_t; __extension__ typedef __quad_t __off64_t; $ gcc -E -P -m32 - <<< "#include <unistd.h>" |& grep -E "typedef .* __quad_t" __extension__ typedef long long int __quad_t; $ gcc -o test -x c - <<< $'#include <stdio.h>\nint main(){printf("sizeof(__off_t) = %zd\\nsizeof(__off64_t) = %zd\\n", sizeof(__off_t), sizeof(__off64_t));}' $ ./test sizeof(__off_t) = 8 sizeof(__off64_t) = 8 $ gcc -m32 -o test -x c - <<< $'#include <stdio.h>\nint main(){printf("sizeof(__off_t) = %zd\\nsizeof(__off64_t) = %zd\\n", sizeof(__off_t), sizeof(__off64_t));}' $ ./test sizeof(__off_t) = 4 sizeof(__off64_t) = 8 With _FILE_OFFSET_BITS=64, pread() is redirection to pread64(): $ gcc -E -P - <<< "#include <unistd.h>" | grep -EA1 "(pread|pwrite)" extern ssize_t pread (int __fd, void *__buf, size_t __nbytes, __off_t __offset) ; extern ssize_t pwrite (int __fd, const void *__buf, size_t __n, __off_t __offset) ; $ gcc -E -P -D_LARGEFILE64_SOURCE - <<< "#include <unistd.h>" | grep -EA1 "(pread|pwrite)" extern ssize_t pread (int __fd, void *__buf, size_t __nbytes, __off_t __offset) ; extern ssize_t pwrite (int __fd, const void *__buf, size_t __n, __off_t __offset) ; extern ssize_t pread64 (int __fd, void *__buf, size_t __nbytes, __off64_t __offset) ; extern ssize_t pwrite64 (int __fd, const void *__buf, size_t __n, __off64_t __offset) ; $ gcc -E -P -D_FILE_OFFSET_BITS=64 - <<< "#include <unistd.h>" | grep -E "(pread|pwrite)" extern ssize_t pread (int __fd, void *__buf, size_t __nbytes, __off64_t __offset) __asm__ ("" "pread64") ; extern ssize_t pwrite (int __fd, const void *__buf, size_t __nbytes, __off64_t __offset) __asm__ ("" "pwrite64") ; $ gcc -E -P -D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 - <<< "#include <unistd.h>" | grep -EA1 "(pread|pwrite)" extern ssize_t pread (int __fd, void *__buf, size_t __nbytes, __off64_t __offset) __asm__ ("" "pread64") ; extern ssize_t pwrite (int __fd, const void *__buf, size_t __nbytes, __off64_t __offset) __asm__ ("" "pwrite64") ; extern ssize_t pread64 (int __fd, void *__buf, size_t __nbytes, __off64_t __offset) ; extern ssize_t pwrite64 (int __fd, const void *__buf, size_t __n, __off64_t __offset) ; $ gcc -S -D_FILE_OFFSET_BITS=64 -o - -x c - <<< $'#include <unistd.h>\nint main(){pread(0,0,0,0);pwrite(0,0,0,0);}' | grep -E "(pread|pwrite)" call pread64 call pwrite64 $ gcc -S -m32 -D_FILE_OFFSET_BITS=64 -o - -x c - <<< $'#include <unistd.h>\nint main(){pread(0,0,0,0);pwrite(0,0,0,0);}' | grep -E "(pread|pwrite)" call pread64 call pwrite64 src/sqliteInt.h of SQLite 3.8.10.1: #ifndef SQLITE_DISABLE_LFS # define _LARGE_FILE 1 # ifndef _FILE_OFFSET_BITS # define _FILE_OFFSET_BITS 64 # endif # define _LARGEFILE_SOURCE 1 #endif After building of SQLite on 64-bit architecture with -DUSE_PREAD and without -DUSE_PREAD64: $ readelf -s .libs/libsqlite3.so | grep -E "(pread|pwrite)" 60: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pread64@GLIBC_2.2.5 (3) 75: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pwrite64@GLIBC_2.2.5 (3) 1619: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pread64@@GLIBC_2.2.5 1651: 0000000000000000 0 FUNC GLOBAL DEFAULT UND pwrite64@@GLIBC_2.2.5
pread64() and pwrite64() are declared when _LARGEFILE64_SOURCE is defined. src/sqliteInt.h of SQLite 3.8.10.1: #if defined(__GNUC__) && !defined(_GNU_SOURCE) # define _GNU_SOURCE #endif features.h of glibc 2.20: #ifdef _GNU_SOURCE ... # undef _LARGEFILE64_SOURCE # define _LARGEFILE64_SOURCE 1 ... #endif So using of pread64() and pwrite64() with glibc and compiler not defining __GNUC__ might require using -D_LARGEFILE64_SOURCE (potentially using append-lfs-flags() from flag-o-matic.eclass).
Fixed in dev-db/sqlite-3.8.10.1.
