Merge pull request #18 from dev-diaries41/staging

Merge v1.1.1 changes to main

Author: dev-diaries41

CHANGELOG.md

@@ -1,3 +1,16 @@
+### v1.1.1 - 02/11/2025
+
+### Added
+* Added new text embedding provider, Mini-LM
+
+### Changed
+* IEmbeddingProvider is require to provider `embeddingDim` variable (used to be optional)
+* Renamed `embeddingLength` to `embeddingDim` for `FileEmbeddingStore` constructor param
+* Move interfaces:
+  - Moved to core/embeddings: `IEmbeddingStore`, `IRetriever`, `IEmbeddingProvider`
+  - Moved to core/processor: `IProcessorListener`
+  - Moved to ml/models: `IModelLoader`
+
 ### v1.1.0 - 30/10/2025
 
 ### Changed

README.md

@@ -3,6 +3,7 @@
 ## Table of Contents
 
 * [Overview](#overview)
+* [Quick Start](#quick-start)
 * [Documentation](docs/README.md)
 * [Key Structure](#key-structure)
 * [Installation](#installation)
@@ -11,7 +12,7 @@
   + [2. Install ML Module (Optional)](#2-install-ml-module-optional)
 * [Design Choices](#design-choices)
 
-  + [Core and ML](#core-and-ml)``
+  + [Core and ML](#core-and-ml)
   + [Constraints](#constraints)
   + [Model](#model)
   + [Embedding Storage](#embedding-storage)
@@ -35,7 +36,125 @@ SmartScanSdk is a modular Android SDK that powers the **SmartScan app**. It prov
 
 ---
 
-## **Key Structure**
+## Quick Start
+
+Below is information on how to get started with embedding, indexing, and searching.
+
+### Embeddings
+
+#### Text Embeddings
+
+Generate vector embeddings from text strings or batches of text for tasks such as semantic search or similarity comparison.
+
+**Usage Example:**
+
+```kotlin
+//import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipTextEmbedder
+
+// Requires model to be in raw resources at e.g res/raw/text_encoder_quant_int8.onnx 
+val textEmbedder = ClipTextEmbedder(context, ResourceId(R.raw.text_encoder_quant_int8))
+val text = "Hello smartscan"
+val embedding = textEmbedder.embed(text)
+
+```
+
+**Batch Example:**
+
+```kotlin
+val texts = listOf("first sentence", "second sentence")
+val embeddings = textEmbedder.embedBatch(texts)
+```
+
+---
+
+#### Image Embeddings
+
+Generate vector embeddings from images (as `Bitmap`) for visual search or similarity tasks.
+
+**Usage Example**
+
+```kotlin
+//import com.fpf.smartscansdk.ml.models.providers.embeddings.clip.ClipImageEmbedder
+
+// Requires model to be in raw resources at e.g res/raw/image_encoder_quant_int8.onnx 
+val imageEmbedder = ClipImageEmbedder(context, ResourceId(R.raw.image_encoder_quant_int8))
+
+val embedding = imageEmbedder.embed(bitmap)
+
+```
+
+
+**Batch Example:**
+
+```kotlin
+val images: List<Bitmap> = ...
+val embeddings = imageEmbedder.embedBatch(images)
+```
+
+### Indexing
+
+To get started with indexing media quickly, you can use the provided `ImageIndex` and `VideoIndexer` classes as shown below. You can optionally create your own indexers (including for text related data) by implementing the `BatchProcessor` interface. See docs for more details.
+
+#### Image Indexing
+
+Index images to enable similarity search. The index is saved as a binary file and managed with a FileEmbeddingStore.
+> **Important**: During indexing the MediaStore Id is used to as the id in the `Embedding` which is stored. This can later be used for retrieval.
+
+
+```kotlin
+val imageEmbedder = ClipImageEmbedder(context, ResourceId(R.raw.image_encoder_quant_int8))
+val imageStore = FileEmbeddingStore(File(context.filesDir, "image_index.bin"), imageEmbedder.embeddingDim, useCache = false) // cache not needed for indexing
+val imageIndexer = ImageIndexer(imageEmbedder, context=context, listener = null, store = imageStore) //optionally pass a listener to handle events
+val ids = getImageIds() // placeholder function to get MediaStore image ids
+imageIndexer.run(ids)
+```
+
+#### Video Indexing
+
+Index videos to enable similarity search. The index is saved as a binary file and managed with a FileEmbeddingStore.
+
+```kotlin
+val imageEmbedder = ClipImageEmbedder(context, ResourceId(R.raw.image_encoder_quant_int8))
+val videoStore = FileEmbeddingStore(File(context.filesDir,  "video_index.bin"), imageEmbedder.embeddingDim, useCache = false )
+val videoIndexer = VideoIndexer(imageEmbedder, context=context, listener = null, store = videoStore, width = ClipConfig.IMAGE_SIZE_X, height = ClipConfig.IMAGE_SIZE_Y)
+val ids = getVideoIds() // placeholder function to get MediaStore video ids
+videoIndexer.run(ids)
+```
+
+### Searching
+
+Below shows how to search using both text queries and an image. The returns results are List<Embedding>. You can use the id from each one, which corresponds to the MediaStore id, to retrieve the result images.
+
+#### Text-to-Image Search
+
+ ```kotlin
+val imageStore = FileEmbeddingStore(File(context.filesDir, "image_index.bin"), imageEmbedder.embeddingDim, useCache = false) // cache not needed for indexing
+val imageRetriever = FileEmbeddingRetriever(imageStore)
+val textEmbedder = ClipTextEmbedder(context, ResourceId(R.raw.text_encoder_quant_int8))
+val query = "my search query"
+val embedding = textEmbedder.embed(query)
+val topK = 20
+val similarityThreshold = 0.2f
+val results = retriever.query(embedding, topK, similarityThreshold)
+
+```
+
+#### Reverse Image Search
+
+```kotlin
+val imageStore = FileEmbeddingStore(File(context.filesDir, "image_index.bin"), imageEmbedder.embeddingDim, useCache = false) // cache not needed for indexing
+val imageRetriever = FileEmbeddingRetriever(imageStore)
+val imageEmbedder = ClipImageEmbedder(context, ResourceId(R.raw.image_encoder_quant_int8))
+val embedding = imageEmbedder.embed(bitmap)
+val topK = 20
+val similarityThreshold = 0.2f
+val results = retriever.query(embedding, topK, similarityThreshold)
+
+```
+
+---
+
+## Key Structure
 
 ```
 SmartScanSdk/
@@ -66,7 +185,7 @@ SmartScanSdk/
 
  ---
 
-## **Installation**
+## Installation
 
 ### **1. Install Core Module**
 
@@ -84,7 +203,7 @@ implementation("com.github.dev-diaries41.smartscan-sdk:smartscan-ml:1.1.0")
 
 ---
 
-## **Design Choices**
+## Design Choices
 
 ### Core and ML
 
@@ -140,7 +259,7 @@ File-based memory-mapped loading is significantly faster and scales better.
 
 ___
 
-## **Gradle / Kotlin Setup Notes**
+## Gradle / Kotlin Setup Notes
 
 * Java 17 / Kotlin JVM 17
 * `compileSdk = 36`, `targetSdk = 34`, `minSdk = 30`

core/src/main/java/com/fpf/smartscansdk/core/data/Embedings.kt

@@ -1,8 +1,6 @@
 package com.fpf.smartscansdk.core.data
 
 
-import android.graphics.Bitmap
-
 // `Embedding` represents a raw vector for a single media item, with `id` corresponding to its `MediaStoreId`.
 data class Embedding(
     val id: Long,
@@ -18,33 +16,5 @@ data class PrototypeEmbedding(
 )
 
 
-interface IEmbeddingStore {
-    val isCached: Boolean
-    val exists: Boolean
-    suspend fun add(newEmbeddings: List<Embedding>)
-    suspend fun remove(ids: List<Long>)
-    suspend fun get(): List<Embedding>
-    fun clear()
-}
-
-interface IRetriever {
-    suspend fun query(
-        embedding: FloatArray,
-        topK: Int,
-        threshold: Float
-    ): List<Embedding>
-}
-
-
-interface IEmbeddingProvider<T> {
-    val embeddingDim: Int? get() = null
-    fun closeSession() = Unit
-    suspend fun embed(data: T): FloatArray
-    suspend fun embedBatch(data: List<T>): List<FloatArray>
-}
-
-
-typealias TextEmbeddingProvider = IEmbeddingProvider<String>
-typealias ImageEmbeddingProvider = IEmbeddingProvider<Bitmap>
 
 

core/src/main/java/com/fpf/smartscansdk/core/data/Processors.kt

@@ -1,15 +1,5 @@
 package com.fpf.smartscansdk.core.data
 
-import android.content.Context
-
-interface IProcessorListener<Input, Output> {
-    suspend fun onActive(context: Context) = Unit
-    suspend fun onBatchComplete(context: Context, batch: List<Output>) = Unit
-    suspend fun onComplete(context: Context, metrics: Metrics.Success) = Unit
-    suspend fun onProgress(context: Context, progress: Float) = Unit
-    fun onError(context: Context, error: Exception, item: Input) = Unit
-    suspend fun onFail(context: Context, failureMetrics: Metrics.Failure) = Unit
-}
 
 sealed class Metrics {
     data class Success(val totalProcessed: Int = 0, val timeElapsed: Long = 0L) : Metrics()

core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingRetriever.kt

@@ -1,7 +1,6 @@
 package com.fpf.smartscansdk.core.embeddings
 
 import com.fpf.smartscansdk.core.data.Embedding
-import com.fpf.smartscansdk.core.data.IRetriever
 
 class FileEmbeddingRetriever(
     private val store: FileEmbeddingStore

core/src/main/java/com/fpf/smartscansdk/core/embeddings/FileEmbeddingStore.kt

@@ -2,7 +2,6 @@ package com.fpf.smartscansdk.core.embeddings
 
 import android.util.Log
 import com.fpf.smartscansdk.core.data.Embedding
-import com.fpf.smartscansdk.core.data.IEmbeddingStore
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.withContext
 import java.io.File
@@ -16,7 +15,7 @@ import java.nio.channels.FileChannel
 
 class FileEmbeddingStore(
     private val file: File,
-    private val embeddingLength: Int,
+    private val embeddingDimension: Int,
     val useCache: Boolean = true,
     ):
     IEmbeddingStore {
@@ -54,12 +53,12 @@ class FileEmbeddingStore(
                 val batch = embeddingsList.subList(index, end)
 
                 // Allocate a smaller buffer for this batch
-                val batchBuffer = ByteBuffer.allocate(batch.size * (8 + 8 + embeddingLength * 4))
+                val batchBuffer = ByteBuffer.allocate(batch.size * (8 + 8 + embeddingDimension * 4))
                     .order(ByteOrder.LITTLE_ENDIAN)
 
                 for (embedding in batch) {
-                    if (embedding.embeddings.size != embeddingLength) {
-                        throw IllegalArgumentException("Embedding length must be $embeddingLength")
+                    if (embedding.embeddings.size != embeddingDimension) {
+                        throw IllegalArgumentException("Embedding dimension must be $embeddingDimension")
                     }
                     batchBuffer.putLong(embedding.id)
                     batchBuffer.putLong(embedding.date)
@@ -89,10 +88,10 @@ class FileEmbeddingStore(
             repeat(count) {
                 val id = buffer.long
                 val date = buffer.long
-                val floats = FloatArray(embeddingLength)
+                val floats = FloatArray(embeddingDimension)
                 val fb = buffer.asFloatBuffer()
                 fb.get(floats)
-                buffer.position(buffer.position() + embeddingLength * 4)
+                buffer.position(buffer.position() + embeddingDimension * 4)
                 map[id] = Embedding(id, date, floats)
             }
             if (useCache) cache = map
@@ -122,7 +121,7 @@ class FileEmbeddingStore(
             val existingCount = headerBuf.int
 
             // Basic validation: each existing entry is at least id(8)+date(8)+EMBEDDING_LEN*4
-            val minEntryBytes = 8 + 8 + embeddingLength * 4
+            val minEntryBytes = 8 + 8 + embeddingDimension * 4
             val maxCountFromSize = (channel.size() / minEntryBytes).toInt()
             if (existingCount < 0 || existingCount > maxCountFromSize + 10_000) {
                 throw IOException("Corrupt embeddings header: count=$existingCount, fileSize=${channel.size()}")
@@ -140,10 +139,10 @@ class FileEmbeddingStore(
             channel.position(channel.size())
 
             for (embedding in newEmbeddings) {
-                if (embedding.embeddings.size != embeddingLength) {
-                    throw IllegalArgumentException("Embedding length must be $embeddingLength")
+                if (embedding.embeddings.size != embeddingDimension) {
+                    throw IllegalArgumentException("Embedding dimension must be $embeddingDimension")
                 }
-                val entryBytes = (8 + 8) + embeddingLength * 4
+                val entryBytes = (8 + 8) + embeddingDimension * 4
                 val buf = ByteBuffer.allocate(entryBytes).order(ByteOrder.LITTLE_ENDIAN)
                 buf.putLong(embedding.id)
                 buf.putLong(embedding.date)

core/src/main/java/com/fpf/smartscansdk/core/embeddings/IEmbeddingProvider.kt

@@ -0,0 +1,16 @@
+package com.fpf.smartscansdk.core.embeddings
+
+import android.graphics.Bitmap
+
+
+interface IEmbeddingProvider<T> {
+    val embeddingDim: Int
+    fun closeSession() = Unit
+    suspend fun embed(data: T): FloatArray
+    suspend fun embedBatch(data: List<T>): List<FloatArray>
+}
+
+
+typealias TextEmbeddingProvider = IEmbeddingProvider<String>
+typealias ImageEmbeddingProvider = IEmbeddingProvider<Bitmap>
+

core/src/main/java/com/fpf/smartscansdk/core/embeddings/IEmbeddingStore.kt

@@ -0,0 +1,12 @@
+package com.fpf.smartscansdk.core.embeddings
+
+import com.fpf.smartscansdk.core.data.Embedding
+
+interface IEmbeddingStore {
+    val isCached: Boolean
+    val exists: Boolean
+    suspend fun add(newEmbeddings: List<Embedding>)
+    suspend fun remove(ids: List<Long>)
+    suspend fun get(): List<Embedding>
+    fun clear()
+}

core/src/main/java/com/fpf/smartscansdk/core/embeddings/IRetriever.kt

@@ -0,0 +1,12 @@
+package com.fpf.smartscansdk.core.embeddings
+
+import com.fpf.smartscansdk.core.data.Embedding
+
+interface IRetriever {
+    suspend fun query(
+        embedding: FloatArray,
+        topK: Int,
+        threshold: Float
+    ): List<Embedding>
+}
+

core/src/main/java/com/fpf/smartscansdk/core/indexers/ImageIndexer.kt

@@ -4,12 +4,12 @@ import android.content.ContentUris
 import android.content.Context
 import android.provider.MediaStore
 import com.fpf.smartscansdk.core.data.Embedding
-import com.fpf.smartscansdk.core.data.IEmbeddingStore
-import com.fpf.smartscansdk.core.data.IProcessorListener
-import com.fpf.smartscansdk.core.data.ImageEmbeddingProvider
 import com.fpf.smartscansdk.core.data.ProcessOptions
+import com.fpf.smartscansdk.core.embeddings.IEmbeddingStore
+import com.fpf.smartscansdk.core.embeddings.ImageEmbeddingProvider
 import com.fpf.smartscansdk.core.media.getBitmapFromUri
 import com.fpf.smartscansdk.core.processors.BatchProcessor
+import com.fpf.smartscansdk.core.processors.IProcessorListener
 import kotlinx.coroutines.NonCancellable
 import kotlinx.coroutines.withContext