Merge branch 'master' of github.com:lemire/simdcomp

Author: lemire

CHANGELOG

@@ -1 +1,3 @@
+Version 0.0.2 (6 February 2014)
+  - added go demo
 Version 0.0.1  (5 February 2014)

README.md

@@ -1,20 +1,34 @@
 The SIMDComp library 
 ====================
 
-A simple C library for compressing lists of integers
+A simple C library for compressing lists of integers using binary packing and SIMD instructions.
+
+This library can decode billions of compressed integers per second on most
+desktop or laptop processors.
+
+What is it for?
+-------------
+
+This is a low-level library for fast integer compression. By design it does not define a compressed
+format. It is up to the (sophisticated) user to create a compressed format.
 
 Requirements
-=============
+-------------
 
 - Your processor should support SSE2 (Pentium4 or better)
 - C99 compliant compiler (GCC is assumed)
 - A Linux-like distribution is assumed by the makefile
 
 Usage
-=======
+-------
 
 Compression works over blocks of 128 integers.
 
+For a complete working example, see example.c (you can build it and
+run it with "make example; ./example").
+
+
+
 1) Lists of integers in random order.
 
         const uint32_t b = maxbits(datain);// computes bit width
@@ -33,7 +47,8 @@ We used differential coding: we store the difference between successive integers
         simdunpackd1(offset, buffer, backbuffer, b1);//uncompressed
 
 Setup
-=======
+---------
+
 
 make
 make test
@@ -43,12 +58,23 @@ and if you are daring:
 make install 
 
 Go
-===
+--------
 
 If you are a go user, there is a "go" folder where you will find a simple demo.
 
+Other libraries
+----------------
+
+FastPFOR is a C++ research library well suited to compress unsorted arrays:
+https://github.com/lemire/FastPFor
+
+SIMDCompressionAndIntersection is a C++ research library well suited for sorted arrays (differential coding)
+and computing intersections:
+https://github.com/lemire/SIMDCompressionAndIntersection
+
 References
-===========
+------------
+
 
 Daniel Lemire and Leonid Boytsov, Decoding billions of integers per second through vectorization, Software: Practice & Experience, 2013. 
 http://dx.doi.org/10.1002/spe.2203

example.c

@@ -0,0 +1,63 @@
+#include <stdio.h>
+#include <time.h>
+#include "simdcomp.h"
+
+
+// compresses data from datain to buffer, returns how many bytes written
+size_t compress(uint32_t * datain, size_t length, uint8_t * buffer) {
+	if(length/SIMDBlockSize*SIMDBlockSize != length) {
+	    printf("Data length should be a multiple of %i \n",SIMDBlockSize);
+	}
+    uint32_t offset = 0;
+    uint8_t * initout = buffer;
+	for(size_t k = 0; k < length / SIMDBlockSize; ++k) {
+        uint32_t b = simdmaxbitsd1(offset,
+                    datain + k * SIMDBlockSize);
+		*buffer++ = b;
+		simdpackwithoutmaskd1(offset, datain + k * SIMDBlockSize, (__m128i *) buffer,
+                    b);
+        offset = datain[k * SIMDBlockSize + SIMDBlockSize - 1];
+        buffer += b * sizeof(__m128i);
+	}
+	return buffer - initout;
+}
+
+
+int main() {
+    int N = 5000 * SIMDBlockSize;//SIMDBlockSize is 128
+    uint32_t * datain = malloc(N * sizeof(uint32_t));
+    size_t compsize;
+    clock_t start, end;
+
+    uint8_t * buffer = malloc(N * sizeof(uint32_t) + N / SIMDBlockSize); // output buffer
+    uint32_t * backbuffer = malloc(SIMDBlockSize * sizeof(uint32_t));
+    for (int gap = 1; gap <= 243; gap *= 3) {
+    	printf("\n");
+        printf(" gap = %u \n", gap);
+        for (int k = 0; k < N; ++k)
+            datain[k] = k * gap;
+        uint32_t offset = 0;
+        compsize = compress(datain,N,buffer);
+        printf("compression rate = %f \n",  (N * sizeof(uint32_t))/ (compsize * 1.0 ));
+        uint8_t * decbuffer = buffer;
+        start = clock();
+        uint32_t bogus = 0;
+        for (int k = 0; k * SIMDBlockSize < N; ++k) {
+        	uint8_t b = *decbuffer++;
+            simdunpackd1(offset, (__m128i *) decbuffer, backbuffer, b);
+            // do something here with backbuffer
+            bogus += backbuffer[3];
+            decbuffer += b * sizeof(__m128i);
+            offset = backbuffer[SIMDBlockSize - 1];
+        }
+        end = clock();
+        double numberofseconds = (end-start)/(double)CLOCKS_PER_SEC;
+        printf("decoding speed in million of integers per second %f \n",N/(numberofseconds*1000.0*1000.0));
+        printf("ignore me %i \n",bogus);
+    }
+    free(buffer);
+    free(datain);
+    free(backbuffer);
+    return 0;
+}
+

include/simdbitpacking.h

@@ -7,11 +7,14 @@
 #include <emmintrin.h>// SSE2 is required
 #include <stdint.h> // use a C99-compliant compiler, please
 #include <string.h> // for memset
-//reads 128 values in in, writes  bit values from out
+
+//reads 128 values from "in", writes  "bit" 128-bit vectors to "out"
 void simdpack(const uint32_t *  in,__m128i *  out, uint32_t bit);
-//reads 128 values in in, writes  bit values from out
+
+//reads 128 values from "in", writes  "bit" 128-bit vectors to "out"
 void simdpackwithoutmask(const uint32_t *  in,__m128i *  out, uint32_t bit);
-//reads bit values in in, writes  128 values to out
+
+//reads  "bit" 128-bit vectors from "in", writes  128 values to "out"
 void simdunpack(const __m128i *  in,uint32_t *  out, uint32_t bit);
 
 

include/simdintegratedbitpacking.h

@@ -9,11 +9,18 @@
 #include <stdint.h> // use a C99-compliant compiler, please
 
 #include "simdcomputil.h"
-//reads 128 values in in, writes  bit values from out
+
+//reads 128 values from "in", writes  "bit" 128-bit vectors to "out"
+// integer values should be in sorted order (for best results)
 void simdpackd1(uint32_t initvalue, const uint32_t *  in,__m128i *  out, uint32_t bit);
-//reads 128 values in in, writes  bit values from out
+
+
+//reads 128 values from "in", writes  "bit" 128-bit vectors to "out"
+// integer values should be in sorted order (for best results)
 void simdpackwithoutmaskd1(uint32_t initvalue, const uint32_t *  in,__m128i *  out, uint32_t bit);
-//reads bit values in in, writes  128 values to out
+
+
+//reads "bit" 128-bit vectors from "in", writes  128 values to "out"
 void simdunpackd1(uint32_t initvalue, const __m128i *  in,uint32_t *  out, uint32_t bit);
 
 

makefile

@@ -42,6 +42,9 @@ simdbitpacking.o: ./src/simdbitpacking.c $(HEADERS)
 simdintegratedbitpacking.o: ./src/simdintegratedbitpacking.c  $(HEADERS)
 	$(CC) $(CFLAGS) -c ./src/simdintegratedbitpacking.c -Iinclude  
 
+example: ./example.c    $(HEADERS) $(OBJECTS)
+	$(CC) $(CFLAGS) -o example ./example.c -Iinclude  $(OBJECTS)
+
 unit: ./src/unit.c    $(HEADERS) $(OBJECTS)
 	$(CC) $(CFLAGS) -o unit ./src/unit.c -Iinclude  $(OBJECTS)
 dynunit: ./src/unit.c    $(HEADERS) $(LIBNAME)

src/unit.c

@@ -4,6 +4,7 @@
 #include <stdio.h>
 #include "simdcomp.h"
 
+
 int main() {
     int N = 5000 * SIMDBlockSize;
     __m128i * buffer = malloc(SIMDBlockSize * sizeof(uint32_t));
@@ -15,20 +16,34 @@ int main() {
             datain[k] = k * gap;
         uint32_t offset = 0;
         for (int k = 0; k * SIMDBlockSize < N; ++k) {
+            /////////////////////////////
+            // First part works for general arrays (sorted or unsorted)
+            /////////////////////////////
+       	    // we compute the bit width
             const uint32_t b = maxbits(datain + k * SIMDBlockSize);
-            simdpackwithoutmask(datain + k * SIMDBlockSize, buffer, b);//compressed
+            // we read 128 integers at "datain + k * SIMDBlockSize" and
+            // write b 128-bit vectors at "buffer"
+            simdpackwithoutmask(datain + k * SIMDBlockSize, buffer, b);
+            // we read back b1 128-bit vectors at "buffer" and write 128 integers at backbuffer
             simdunpack(buffer, backbuffer, b);//uncompressed
             for (int j = 0; j < SIMDBlockSize; ++j) {
                 if (backbuffer[j] != datain[k * SIMDBlockSize + j]) {
                     printf("bug in simdpack\n");
                     return -2;
                 }
             }
+            /////////////////////////////
+            // next part assumes that the data is sorted (uses differential coding)
+            /////////////////////////////
+            // we compute the bit width
             const uint32_t b1 = simdmaxbitsd1(offset,
                     datain + k * SIMDBlockSize);
+            // we read 128 integers at "datain + k * SIMDBlockSize" and
+            // write b1 128-bit vectors at "buffer"
             simdpackwithoutmaskd1(offset, datain + k * SIMDBlockSize, buffer,
-                    b1);//compressed
-            simdunpackd1(offset, buffer, backbuffer, b1);//uncompressed
+                    b1);
+            // we read back b1 128-bit vectors at "buffer" and write 128 integers at backbuffer
+            simdunpackd1(offset, buffer, backbuffer, b1);
             for (int j = 0; j < SIMDBlockSize; ++j) {
                 if (backbuffer[j] != datain[k * SIMDBlockSize + j]) {
                     printf("bug in simdpack d1\n");