s3 libcurl test and docker fixes

.github/workflows/rust.yml

@@ -68,14 +68,26 @@ jobs:
       - name: Run cargo-tarpaulin
         uses: actions-rs/tarpaulin@v0.1
         with:
-          args: '--out Lcov -- --test-threads 1'
+          args: '--out Lcov --exclude-files hts-sys/*_prebuilt_bindings.rs -- --test-threads 1'
 
       - name: Upload coverage
         uses: coverallsapp/github-action@master
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           path-to-lcov: ./lcov.info
 
+      - name: Test standard build without default features
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --no-default-features
+
+      - name: Test standard build with all features
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --all-features
+
       - name: Test musl build without default features
         env:
            CFLAGS: -I/usr/local/musl/include
@@ -92,5 +104,4 @@ jobs:
         with:
           use-cross: true  
           command: build
-          args: --target x86_64-unknown-linux-musl --all-features
-
+          args: --target x86_64-unknown-linux-musl --all-features

Cargo.toml

@@ -35,9 +35,13 @@ hts-sys = { version = "^1.10", path = "hts-sys", default-features = false }
 default = ["bzip2", "lzma", "curl"]
 bzip2 = ["hts-sys/bzip2"]
 lzma = ["hts-sys/lzma"]
+bindgen = ["hts-sys/bindgen"]
 curl = ["hts-sys/curl"]
+libdeflate = ["hts-sys/libdeflate-sys"]
+s3 = ["hts-sys/s3"]
+gcs = ["hts-sys/gcs"]
+static = ["hts-sys/static"]
 serde = ["serde_base", "serde_bytes"]
-static = []
 
 [dev-dependencies]
 tempdir = "0.3"

Cross.toml

@@ -7,6 +7,6 @@ passthrough = [
 
 
 [target.x86_64-unknown-linux-musl]
-image = "brainstorm/cross-x86_64-unknown-linux-musl:1.0.0"
+image = "brainstorm/cross-x86_64-unknown-linux-musl:1.0.2"
 [target.x86_64-unknown-linux-gnu]
 image = "brainstorm/cross-x86_64-unknown-linux-gnu:1.0.0"

README.md

@@ -19,7 +19,11 @@ If you only want to use the library, there is no need to clone the repository. G
 
 ## Requirements
 
-To compile this crate you need docker and cross:
+rust-htslib comes with pre-built bindings to htslib for Mac and Linux. You will need a C toolchain compatible with the `cc` crate. The build script for this crate will automatically build a link htslib.
+
+
+### MUSL build
+To compile this for MUSL crate you need docker and cross:
 
 ```shell
 $ cargo install cross
@@ -62,7 +66,16 @@ with these compression methods, you can deactivate these features to reduce you
 rust-htslib = { version = "*", default-features = false }
 ```
 
-`rust-htslib` also has optional support for `serde`, to allow (de)serialization of `bam::Record` via any serde-supported format:
+`rust-htslib` has optional support for `serde`, to allow (de)serialization of `bam::Record` via any serde-supported format.
+
+Http access to files is available with the `curl` feature.
+
+Beta-level S3 and Google Cloud Storge support is available with the `s3` and `gcs` features.
+
+`rust-htslib` can optionally use `bindgen` to generate bindings to htslib. This can slow down the build substantially. Enabling the `bindgen` feature will 
+cause `hts-sys` to use a create a binding file for your architecture. Pre-built bindings are supplied for Mac and Linux. The `bindgen` feature on Windows is untested - please file a bug if you need help.
+
+
 
 ```toml
 [dependencies]

docker/Dockerfile.musl

@@ -9,6 +9,7 @@ ENV CURL_VERSION 7.70.0
 
 ENV LIBCLANG_PATH /usr/lib/llvm-10/lib
 ENV LLVM_CONFIG_PATH /usr/bin
+ENV LIBDIR /usr/local/musl/lib
 
 
 # The default includes and packages from the Ubuntu distro that cross uses will generate all sorts of linux-headers related include errors, see:
@@ -38,10 +39,15 @@ WORKDIR /root
 RUN wget https://github.com/richfelker/musl-cross-make/archive/v$MUSL_CROSS_MAKE_VERSION.tar.gz \
     && tar xvfz v$MUSL_CROSS_MAKE_VERSION.tar.gz
 WORKDIR /root/musl-cross-make-$MUSL_CROSS_MAKE_VERSION
-COPY config-musl-cross-make.mak config.mak
+COPY conf/config-musl-cross-make.mak config.mak
 RUN make -j40 install && cd .. && rm v$MUSL_CROSS_MAKE_VERSION.tar.gz
 			       #&& rm -rf musl-cross-make-*
 
+# Make sure the linker finds the object it needs
+RUN rm /etc/ld.so.conf.d/*
+COPY conf/x86_64-linux-musl.conf /etc/ld.so.conf.d/x86_64-linux-musl.conf
+RUN ln -sf /etc/ld.so.conf.d/x86_64-linux-musl.conf /etc/ld.so.conf.d/x86_64-unknown-linux-musl.conf && ldconfig
+
 # Now we assume we have a properly configured musl-cross...
 ENV PATH "/usr/local/musl/bin:$PATH"
 ENV CFLAGS "-fPIC"
@@ -68,9 +74,8 @@ RUN git clone https://github.com/cloudflare/zlib cloudflare-zlib && \
 #    && make -j40 install
 RUN git clone git://sourceware.org/git/bzip2 \
     && cd bzip2 \
-    && make -j40 CC=$CC AR=$AR RANLIB=$RANLIB CFLAGS=$CFLAGS bzip2 \
-    && make PREFIX=/usr/local/musl -j40 bzip2 \
-    && make -j40 install && cd .. && rm -rf bzip2
+    && make bzip2 -j40 \
+    && make PREFIX=/usr/local/musl install -j40 && cd .. && rm -rf bzip2
 RUN wget https://tukaani.org/xz/xz-$LZMA_VERSION.tar.bz2 \
     && tar xvfj xz-$LZMA_VERSION.tar.bz2 \
     && cd xz-$LZMA_VERSION \
@@ -106,7 +111,7 @@ RUN wget https://curl.haxx.se/download/curl-$CURL_VERSION.tar.gz \
     && tar xvfz curl-$CURL_VERSION.tar.gz && cd curl-$CURL_VERSION \
     && ./configure --prefix=/usr/local/musl --host x86_64-linux-musl \
         --with-ssl=/usr/local/musl --with-zlib=/usr/local/musl --enable-static \
-        --disable-dict --disable-ftp --disable-gopher --disable-imap \
+        --disable-dict --disable-gopher --disable-imap \
         --disable-pop3 --disable-rtsp --disable-smb --disable-smtp --disable-telnet \
         --disable-tftp --disable-ntlm --disable-ldap \
         && make -j40 install && cd .. \
@@ -117,7 +122,7 @@ ENV OPENSSL_DIR /usr/local/musl
 
 # Hack to force ld stick to musl on the hts-sys/build.rs side, otherwise:
 #   = note: /usr/bin/ld: /target/debug/deps/liblibloading-689161fea10b6234.rlib(global_static.o): unable to initialize decompress status for section .debug_info
-RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld
+#RUN rm /usr/bin/ld && ln -sf /usr/local/musl/bin/x86_64-linux-musl-ld /usr/bin/ld
 
 # Prepare rustup and toolchain locally for easy manual intervention in this container
 #RUN wget https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init \

docker/conf/x86_64-linux-musl.conf

@@ -0,0 +1,2 @@
+# MUSL support
+/usr/local/musl/lib

hts-sys/Cargo.toml

@@ -18,20 +18,25 @@ tag-message = "Version {{version}} of Rust-HTSlib."
 
 [dependencies]
 libz-sys = { version = "1.0.25", features = ["static"] }
-# https://github.com/alexcrichton/bzip2-rs/issues/56
 bzip2-sys = { version = "0.1.8", optional = true }
 lzma-sys = { version = "0.1.16", optional = true, features = ["static"] }
-curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "static-ssl"] }
+curl-sys = { version = "0.4.31", optional = true, features = ["static-curl", "static-ssl", "protocol-ftp"] }
+libdeflate-sys = { version = "0.5", optional = true }
+
+[target.'cfg(all(unix, not(target_os = "macos")))'.dependencies]
+openssl-sys = { version = "0.9", optional = true }
 
 [features]
-default = ["bzip2", "lzma"]
+default = ["bzip2", "lzma", "curl"]
 bzip2 = ["bzip2-sys"]
 lzma = ["lzma-sys"]
-curl = ["curl-sys"]
-static = []
+curl = ["curl-sys", "openssl-sys"]
+gcs = ["curl"] # Google Cloud Storage support
+s3 = ["curl"] # Amazon S3 support
+static = [] # Don't dynamically link to other libraries
 
 [build-dependencies]
 fs-utils = "1.1"
-bindgen = { version = "0.53.2", default-features = false, features = ["runtime"] }
-cc = "1.0"
+bindgen = { version = "0.53.2", default-features = false, features = ["runtime"], optional = true }
+cc = { version = "1.0", features = ["parallel"] }
 glob = "0.3.0"