optimize insert

Author: phartenfeller

src/lib/server/db/index.ts

@@ -16,6 +16,15 @@ export type Customer = {
 	fax: string;
 };
 
+export type OrderDetail = {
+	id: number;
+	orderId: number;
+	productId: number;
+	unitPrice: number;
+	quantity: number;
+	discount: number;
+};
+
 type QueryWrapper = {
 	moreRows: boolean;
 };
@@ -49,3 +58,26 @@ export function getCustomers({ offset = 0, limit = 50 }): QueryResult<Customer[]
 		moreRows: getCustomerCount() > offset + limit
 	};
 }
+
+export function getOrderDetailCount(): number {
+	const stmt = db.prepare(`SELECT COUNT(*) as "cnt" FROM "Order Details"`);
+	const data = stmt.get() as { cnt: number };
+	return data.cnt;
+}
+
+export function getOrderDetails({ offset = 0, limit = 50 }): QueryResult<OrderDetail[]> {
+	const stmt = db.prepare(`
+  SELECT ROWID as "id"
+	     , OrderId as "orderId"
+       , ProductId as "productId"
+       , UnitPrice as "unitPrice"
+       , Quantity as "quantity"
+       , Discount as "discount"
+    FROM "Order Details" LIMIT $limit OFFSET $offset`);
+	const data = stmt.all({ limit, offset }) as OrderDetail[];
+
+	return {
+		data,
+		moreRows: getOrderDetailCount() > offset + limit
+	};
+}

src/lib/sqlite/initStorages.ts

@@ -10,7 +10,7 @@ import {
 	type FillStorageResponseData
 } from './types';
 
-const storages = ['customers_v1'];
+const storages = ['customers_v1', 'order_details_v1'];
 
 async function getStructure(storage: string): Promise<TableStructure> {
 	const res = await fetch(`/api/data/${storage}/structure`);
@@ -36,6 +36,8 @@ async function createStorage(storage: string, structure: TableStructure) {
 }
 
 async function fillStorage(storage: string, structure: TableStructure) {
+	console.time(`fillStorage-${storage}`);
+
 	const PAGE_SIZE = 100;
 	let currOffset = 0;
 	let fetchMore = false;
@@ -59,6 +61,8 @@ async function fillStorage(storage: string, structure: TableStructure) {
 		currOffset += PAGE_SIZE;
 		fetchMore = moreRows;
 	} while (fetchMore);
+
+	console.timeEnd(`fillStorage-${storage}`);
 }
 
 export default async function initStorages() {

src/lib/sqlite/worker/storageHandlers.ts

@@ -68,9 +68,21 @@ export function handleFillStorage(
 		const src = genInsertSql(storageId, structure);
 		console.log('Insert sql:', src);
 
-		for (const row of rows) {
-			db.exec({ sql: src, bind: getBindObject(row) });
-		}
+		db.transaction(() => {
+			const stmnt = db.prepare(src);
+
+			for (const row of rows) {
+				try {
+					stmnt.bind(getBindObject(row));
+				} catch (err) {
+					console.error('Error binding row:', row);
+				} finally {
+					stmnt.stepReset();
+				}
+			}
+
+			stmnt.finalize();
+		});
 
 		return {};
 	} catch (err) {

src/routes/api/data/order_details_v1/data/+server.ts

@@ -0,0 +1,16 @@
+import { getOrderDetails } from '$lib/server/db';
+import { error, json, type RequestHandler } from '@sveltejs/kit';
+
+export const GET = (({ url }) => {
+	// https://localhost:5173/api/data/customers_v1/data?offset=0&limit=50
+	const offset = parseInt(url.searchParams.get('offset') ?? '');
+	const limit = parseInt(url.searchParams.get('limit') ?? '');
+
+	if (isNaN(offset) || isNaN(limit)) {
+		console.error('Invalid offset or limit', { offset, limit });
+		throw error(400, 'Invalid offset or limit');
+	}
+
+	const data = getOrderDetails({ offset, limit });
+	return json(data);
+}) satisfies RequestHandler;

src/routes/api/data/order_details_v1/structure/+server.ts

@@ -0,0 +1,19 @@
+import { json, type RequestHandler } from '@sveltejs/kit';
+import type { TableStructure } from '../../types';
+
+export const GET = (() => {
+	// https://localhost:5173/api/data/customers_v1/structure
+
+	const data: TableStructure = {
+		columns: [
+			{ name: 'id', type: 'number' },
+			{ name: 'orderId', type: 'number' },
+			{ name: 'productId', type: 'number' },
+			{ name: 'unitPrice', type: 'number' },
+			{ name: 'quantity', type: 'number' },
+			{ name: 'discount', type: 'number' }
+		],
+		pkColumn: 'id'
+	};
+	return json(data);
+}) satisfies RequestHandler;