Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 9 additions & 10 deletions src/data/datasetDiagnostics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
datasetFileUrlFor,
xeniumManifestFileFor,
} from './datasetFiles'
import { likelyMissingFileError, noteReadableResponse } from './hostResponses'
import { fetchHead, likelyMissingFileError, noteReadableResponse } from './hostResponses'
import { morphologyImagePathsFromExperiment } from './morphologyImage'
import { assertHttpRangeSupport } from './rangeSupport'
import {
Expand Down Expand Up @@ -1144,12 +1144,10 @@ async function probeChunks(
}

try {
const response = await (options.fetcher ?? fetch)(
const response = await fetchHead(
options.fetcher ?? fetch,
datasetFileUrl(url, target.key),
{
method: 'HEAD',
...(options.signal === undefined ? {} : { signal: options.signal }),
},
options.signal === undefined ? {} : { signal: options.signal },
)
results[index] = { ...target, status: response.status }
} catch {
Expand Down Expand Up @@ -1435,10 +1433,11 @@ async function checkHeadFile(
let response: Response

try {
response = await (options.fetcher ?? fetch)(url, {
method: 'HEAD',
...(options.signal === undefined ? {} : { signal: options.signal }),
})
response = await fetchHead(
options.fetcher ?? fetch,
url,
options.signal === undefined ? {} : { signal: options.signal },
)
} catch (cause) {
// A host that hides its error responses (EBI FTP) makes a missing file
// look like a network failure.
Expand Down
6 changes: 2 additions & 4 deletions src/data/gex.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { getWholeArray } from '../zarr/zarrWholeArray'
import { geneExpressionStageTiming } from './geneExpressionTiming'
import { coalesceCellMajorRanges, emptyCellsFor } from './featureBarcodeMatrixH5'
import { createGeneFeatureIndex } from './geneFeatures'
import { fetchHead } from './hostResponses'
import { selectionToSql } from './gexSelection'
import {
columnSlicesReadBytes,
Expand Down Expand Up @@ -1518,10 +1519,7 @@ async function assertFullFileReadWithinBudget(
throw new RangeError(`Full-file matrix byte budget must be non-negative.`)
}

const response = await fetch(url, {
method: 'HEAD',
...(signal === undefined ? {} : { signal }),
})
const response = await fetchHead(fetch, url, signal === undefined ? {} : { signal })

if (!response.ok) {
throw new Error(`cell_feature_matrix.h5 HEAD returned HTTP ${response.status}.`)
Expand Down
108 changes: 107 additions & 1 deletion src/data/hostResponses.test.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { afterEach, describe, expect, it } from 'vitest'
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
clearHostResponses,
fetchHead,
likelyMissingFileError,
noteReadableResponse,
} from './hostResponses'
Expand Down Expand Up @@ -56,3 +57,108 @@ describe('failed requests on hosts that hide error responses', () => {
).toBeUndefined()
})
})

describe('HEAD requests on hosts whose CORS rules allow GET only', () => {
const FILE_URL = 'https://bucket.example.test/store.zarr/X/data/0'

function getOnlyHost(file: Response | (() => Response)) {
const requests: string[] = []
const fetcher = (async (_input: RequestInfo | URL, init?: RequestInit) => {
const method = init?.method ?? 'GET'
requests.push(`${method} ${new Headers(init?.headers).get('Range') ?? ''}`.trim())

if (method === 'HEAD') {
throw new TypeError('Failed to fetch')
}

return typeof file === 'function' ? file() : file
}) as typeof fetch
return { fetcher, requests }
}

it('reads the size from a one-byte Range GET when HEAD fails', async () => {
const { fetcher, requests } = getOnlyHost(
() =>
new Response(new Uint8Array([1]), {
status: 206,
headers: { 'Content-Range': 'bytes 0-0/53246462', 'Content-Length': '1' },
}),
)

const response = await fetchHead(fetcher, FILE_URL)

expect(response.status).toBe(200)
expect(response.ok).toBe(true)
expect(response.headers.get('Content-Length')).toBe('53246462')
expect(response.body).toBeNull()
expect(requests).toEqual(['HEAD', 'GET bytes=0-0'])

// The origin answered GET, so it is not sent HEAD again.
await fetchHead(fetcher, 'https://bucket.example.test/store.zarr/obs/.zattrs')
expect(requests).toEqual(['HEAD', 'GET bytes=0-0', 'GET bytes=0-0'])
})

it('passes on the status of a missing file', async () => {
const { fetcher } = getOnlyHost(() => new Response(null, { status: 404 }))

const response = await fetchHead(fetcher, FILE_URL)

expect(response.status).toBe(404)
expect(response.ok).toBe(false)
})

it('keeps the size of a whole-file answer from a host that ignores Range', async () => {
const { fetcher } = getOnlyHost(
() => new Response('{"zarr_format":2}', { headers: { 'Content-Length': '17' } }),
)

const response = await fetchHead(fetcher, FILE_URL)

expect(response.status).toBe(200)
expect(response.headers.get('Content-Length')).toBe('17')
})

it('throws the HEAD failure when the GET fails too', async () => {
const cause = new TypeError('Failed to fetch')
const fetcher = vi.fn(async () => {
throw cause
}) as unknown as typeof fetch

await expect(fetchHead(fetcher, FILE_URL)).rejects.toBe(cause)
expect(fetcher).toHaveBeenCalledTimes(2)
})

it('does not fall back on an origin that has answered HEAD', async () => {
const fetcher = vi.fn(async (_input: RequestInfo | URL, init?: RequestInit) => {
if (String(_input).endsWith('/missing')) {
throw new TypeError('Failed to fetch')
}

return new Response(null, { status: init?.method === 'HEAD' ? 200 : 206 })
}) as unknown as typeof fetch

await fetchHead(fetcher, 'https://bucket.example.test/present')
await expect(
fetchHead(fetcher, 'https://bucket.example.test/missing'),
).rejects.toThrow('Failed to fetch')
expect(fetcher).toHaveBeenCalledTimes(2)
})

it('does not fall back when a HEAD sent alongside succeeds first', async () => {
let failMissing: (error: Error) => void = () => undefined
const fetcher = vi.fn((input: RequestInfo | URL) =>
String(input).endsWith('/missing')
? new Promise<Response>((_resolve, reject) => {
failMissing = reject
})
: Promise.resolve(new Response(null, { status: 200 })),
) as unknown as typeof fetch

const missing = fetchHead(fetcher, 'https://bucket.example.test/missing')
await fetchHead(fetcher, 'https://bucket.example.test/present')
failMissing(new TypeError('Failed to fetch'))

await expect(missing).rejects.toThrow('Failed to fetch')
expect(fetcher).toHaveBeenCalledTimes(2)
})
})
89 changes: 89 additions & 0 deletions src/data/hostResponses.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ interface OriginResponses {
}

const originResponses = new Map<string, OriginResponses>()
// Whether each origin answers HEAD readably, or only GET.
const originHeadSupport = new Map<string, 'head' | 'get'>()

export class LikelyMissingFileError extends Error {
constructor(message: string, options?: ErrorOptions) {
Expand Down Expand Up @@ -53,8 +55,95 @@ export function likelyMissingFileError(
)
}

// A HEAD request for hosts whose CORS rules allow GET only (the
// azimuth-vitessce S3 bucket): they answer HEAD without CORS headers, so it
// fails like a network error. When HEAD fails on an origin where none has
// succeeded, a one-byte Range GET stands in, returned as a bodiless response
// whose Content-Length is the file's size from Content-Range. An origin that
// answers it is sent that GET from then on instead of HEAD.
export async function fetchHead(
fetcher: typeof fetch,
url: string,
init: RequestInit = {},
): Promise<Response> {
const origin = originOf(url)
const support = origin === undefined ? undefined : originHeadSupport.get(origin)

if (support === 'get') {
return fetchFirstByteAsHead(fetcher, url, init)
}

try {
const response = await fetcher(url, { ...init, method: 'HEAD' })

if (origin !== undefined) {
originHeadSupport.set(origin, 'head')
}

return response
} catch (cause) {
// Read again: a HEAD sent alongside this one may have succeeded since.
if (
origin === undefined ||
originHeadSupport.get(origin) === 'head' ||
!isFailedFetch(cause) ||
init.signal?.aborted === true
) {
throw cause
}

let response: Response

try {
response = await fetchFirstByteAsHead(fetcher, url, init)
} catch {
throw cause
}

originHeadSupport.set(origin, 'get')
return response
}
}

async function fetchFirstByteAsHead(
fetcher: typeof fetch,
url: string,
init: RequestInit,
): Promise<Response> {
const requestHeaders = new Headers(init.headers)
requestHeaders.set('Range', 'bytes=0-0')
const response = await fetcher(url, {
...init,
method: 'GET',
headers: requestHeaders,
cache: 'no-store',
})
// A host that ignores Range sends the whole file.
void response.body?.cancel().catch(() => undefined)

const headers = new Headers(response.headers)
const size = /\/(\d+)\s*$/.exec(response.headers.get('Content-Range') ?? '')?.[1]
// An empty file has no first byte: 416 with `bytes */0`.
const partial = response.status === 206 || (response.status === 416 && size === '0')

if (partial) {
if (size === undefined) {
headers.delete('Content-Length')
} else {
headers.set('Content-Length', size)
}
}

return new Response(null, {
status: partial ? 200 : response.status,
statusText: partial ? 'OK' : response.statusText,
headers,
})
}

export function clearHostResponses(): void {
originResponses.clear()
originHeadSupport.clear()
}

function isFailedFetch(cause: unknown): boolean {
Expand Down
10 changes: 6 additions & 4 deletions src/data/visiumFeatureSlice.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
} from './h5WorkerFile'
import { waitForSharedLoad, type SharedLoad } from '../utils/sharedLoad'
import type { CellFeatureMatrixReadOptions, GexResult } from './gexTypes'
import { fetchHead } from './hostResponses'
import {
createVisiumFeatureSliceCache,
elapsedMs,
Expand Down Expand Up @@ -245,10 +246,11 @@ async function assertFeatureSliceFullFileReadWithinBudget(
throw new RangeError(`Feature Slice full-file budget must be non-negative.`)
}

const response = await (fetcher ?? fetch)(url, {
method: 'HEAD',
...(signal === undefined ? {} : { signal }),
})
const response = await fetchHead(
fetcher ?? fetch,
url,
signal === undefined ? {} : { signal },
)

if (!response.ok) {
throw new Error(`feature_slice.h5 HEAD returned HTTP ${response.status}.`)
Expand Down
11 changes: 7 additions & 4 deletions src/data/zipArchive.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
// entries of up to a few hundred MB. A deflate stream cannot be read from the
// middle, so an entry is always read whole.

import { fetchHead } from './hostResponses'

export interface ZipEntry {
readonly name: string
// 0 stored, 8 deflated.
Expand Down Expand Up @@ -57,10 +59,11 @@ export async function readZipDirectory(
options: ZipReadOptions = {},
): Promise<ZipDirectory> {
const fetcher = options.fetcher ?? globalThis.fetch.bind(globalThis)
const head = await fetcher(url, {
method: 'HEAD',
...(options.signal === undefined ? {} : { signal: options.signal }),
})
const head = await fetchHead(
fetcher,
url,
options.signal === undefined ? {} : { signal: options.signal },
)

if (!head.ok) {
throw new ZipArchiveError(url, `HEAD returned HTTP ${head.status}.`)
Expand Down
Loading