-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
src: implement OriginProvider and S3Provider
Implements the `OriginProvider` and `S3Provider` classes as described in #111. Allows `R2Provider` to take in a fallback provider that will be used if R2 fails.
- Loading branch information
Showing
6 changed files
with
281 additions
and
23 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,98 @@ | ||
| import { CACHE_HEADERS } from '../constants/cache'; | ||
| import { Context } from '../context'; | ||
| import { | ||
| GetFileOptions, | ||
| GetFileResult, | ||
| HeadFileResult, | ||
| HttpResponseHeaders, | ||
| Provider, | ||
| ReadDirectoryResult, | ||
| } from './provider'; | ||
|
|
||
| type OriginProviderCtorOptions = { | ||
| ctx: Context; | ||
| }; | ||
|
|
||
| /** | ||
| * Provides assets from origin.nodejs.org, used as a fallback for if R2 fails. | ||
| */ | ||
| export class OriginProvider implements Provider { | ||
| private ctx: Context; | ||
|
|
||
| constructor({ ctx }: OriginProviderCtorOptions) { | ||
| this.ctx = ctx; | ||
| } | ||
|
|
||
| async headFile(path: string): Promise<HeadFileResult | undefined> { | ||
| const res = await fetch(this.ctx.env.ORIGIN_HOST + path, { | ||
| method: 'HEAD', | ||
| }); | ||
|
|
||
| if (res.status === 404) { | ||
| return undefined; | ||
| } | ||
|
|
||
| return { | ||
| httpStatusCode: res.status, | ||
| httpHeaders: originHeadersToOurHeadersObject(res.headers), | ||
| }; | ||
| } | ||
|
|
||
| async getFile( | ||
| path: string, | ||
| options?: GetFileOptions | undefined | ||
| ): Promise<GetFileResult | undefined> { | ||
| const res = await fetch(this.ctx.env.ORIGIN_HOST + path, { | ||
| headers: { | ||
| 'user-agent': 'release-cloudflare-worker', | ||
| 'if-match': options?.conditionalHeaders?.ifMatch ?? '', | ||
| 'if-none-match': options?.conditionalHeaders?.ifMatch ?? '', | ||
| 'if-modified-since': | ||
| options?.conditionalHeaders?.ifModifiedSince?.toUTCString() ?? '', | ||
| 'if-unmodified-since': | ||
| options?.conditionalHeaders?.ifUnmodifiedSince?.toUTCString() ?? '', | ||
| range: options?.rangeHeader ?? '', | ||
| }, | ||
| }); | ||
|
|
||
| if (res.status === 404) { | ||
| return undefined; | ||
| } | ||
|
|
||
| return { | ||
| contents: res.body, | ||
| httpStatusCode: res.status, | ||
| httpHeaders: originHeadersToOurHeadersObject(res.headers), | ||
| }; | ||
| } | ||
|
|
||
| async readDirectory(path: string): Promise<ReadDirectoryResult | undefined> { | ||
| const res = await fetch(this.ctx.env.ORIGIN_HOST + path); | ||
|
|
||
| if (res.status === 404) { | ||
| return undefined; | ||
| } | ||
|
|
||
| return { | ||
| body: res.body, | ||
| httpStatusCode: res.status, | ||
| httpHeaders: originHeadersToOurHeadersObject(res.headers), | ||
| }; | ||
| } | ||
| } | ||
|
|
||
| function originHeadersToOurHeadersObject( | ||
| headers: Headers | ||
| ): HttpResponseHeaders { | ||
| return { | ||
| etag: headers.get('etag') ?? '', | ||
| 'accept-range': headers.get('accept-range') ?? 'bytes', | ||
| 'access-control-allow-origin': | ||
| headers.get('access-control-allow-origin') ?? '', | ||
| 'cache-control': CACHE_HEADERS.failure, // We don't want to cache these responses | ||
| 'last-modified': headers.get('last-modified') ?? '', | ||
| 'content-language': headers.get('content-language') ?? '', | ||
| 'content-disposition': headers.get('content-disposition') ?? '', | ||
| 'content-length': headers.get('content-length') ?? '0', | ||
| }; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,124 @@ | ||
| import { | ||
| ListObjectsV2Command, | ||
| ListObjectsV2CommandOutput, | ||
| S3Client, | ||
| } from '@aws-sdk/client-s3'; | ||
| import { Context } from '../context'; | ||
| import { | ||
| File, | ||
| GetFileOptions, | ||
| GetFileResult, | ||
| HeadFileResult, | ||
| Provider, | ||
| ReadDirectoryResult, | ||
| } from './provider'; | ||
| import { retryWrapper } from '../utils/provider'; | ||
| import { R2_RETRY_LIMIT, S3_MAX_KEYS } from '../constants/limits'; | ||
|
|
||
| type S3ProviderCtorOptions = { | ||
| ctx: Context; | ||
| fallbackProvider?: Provider; | ||
| }; | ||
|
|
||
| /** | ||
| * This provides assets from an S3-compatible data source. In our case, it's | ||
| * still R2. We use this only for directory listing. In R2's bindings api, | ||
| * there's some internal response size limit that makes us need to send | ||
| * an absurd amount of requests in order to list the full contents of some | ||
| * directories. Using the S3 api was the recommended fix from the R2 team. | ||
| */ | ||
| export class S3Provider implements Provider { | ||
| private ctx: Context; | ||
| private fallbackProvider?: Provider; | ||
| private client: S3Client; | ||
|
|
||
| constructor({ ctx, fallbackProvider }: S3ProviderCtorOptions) { | ||
| this.ctx = ctx; | ||
| this.fallbackProvider = fallbackProvider; | ||
|
|
||
| this.client = new S3Client({ | ||
| region: 'auto', | ||
| endpoint: ctx.env.S3_ENDPOINT, | ||
| credentials: { | ||
| accessKeyId: ctx.env.S3_ACCESS_KEY_ID, | ||
| secretAccessKey: ctx.env.S3_ACCESS_KEY_SECRET, | ||
| }, | ||
| }); | ||
| } | ||
|
|
||
| headFile(_: string): Promise<HeadFileResult | undefined> { | ||
| throw new Error('Method not implemented.'); | ||
| } | ||
|
|
||
| getFile( | ||
| _: string, | ||
| _2?: GetFileOptions | undefined | ||
| ): Promise<GetFileResult | undefined> { | ||
| throw new Error('Method not implemented.'); | ||
| } | ||
|
|
||
| async readDirectory(path: string): Promise<ReadDirectoryResult | undefined> { | ||
| const directories = new Set<string>(); | ||
| let hasIndexHtmlFile = false; | ||
| const files: File[] = []; | ||
|
|
||
| let isTruncated = true; | ||
| let cursor: string | undefined; | ||
| while (isTruncated) { | ||
| let result: ListObjectsV2CommandOutput; | ||
| try { | ||
| result = await retryWrapper( | ||
| async () => { | ||
| return this.client.send( | ||
| new ListObjectsV2Command({ | ||
| Bucket: this.ctx.env.BUCKET_NAME, | ||
| Prefix: path, | ||
| Delimiter: '/', | ||
| MaxKeys: S3_MAX_KEYS, | ||
| ContinuationToken: cursor, | ||
| }) | ||
| ); | ||
| }, | ||
| R2_RETRY_LIMIT, | ||
| this.ctx.sentry | ||
| ); | ||
| } catch (err) { | ||
| if (this.fallbackProvider !== undefined) { | ||
| // Give up and send to the fallback | ||
| return this.fallbackProvider.readDirectory(path); | ||
| } | ||
|
|
||
| throw err; | ||
| } | ||
|
|
||
| result.CommonPrefixes?.forEach(directory => { | ||
| directories.add(directory.Prefix!.substring(path.length)); | ||
| }); | ||
|
|
||
| result.Contents?.forEach(object => { | ||
| if (object.Key!.endsWith('index.html')) { | ||
| hasIndexHtmlFile = true; | ||
| } | ||
|
|
||
| files.push({ | ||
| name: object.Key!.substring(path.length), | ||
| size: object.Size!, | ||
| lastModified: object.LastModified!, | ||
| }); | ||
| }); | ||
|
|
||
| isTruncated = result.IsTruncated ?? false; | ||
| cursor = result.NextContinuationToken; | ||
| } | ||
|
|
||
| if (directories.size === 0 && files.length === 0) { | ||
| return undefined; | ||
| } | ||
|
|
||
| return { | ||
| subdirectories: Array.from(directories), | ||
| hasIndexHtmlFile, | ||
| files, | ||
| }; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters