feat: add S3/R2 storage support for files

.eslintrc

@@ -3,6 +3,7 @@
   "extends": ["@dexaai/eslint-config", "@dexaai/eslint-config/node"],
   "ignorePatterns": ["node_modules", "dist", ".next"],
   "rules": {
+    "no-process-env": "off",
     "no-console": "off",
     "import/order": "off"
   }

package.json

@@ -21,13 +21,16 @@
     "test": "run-p test:*",
     "test:format": "prettier --check \"**/*.{js,ts,tsx}\"",
     "test:lint": "eslint .",
-    "test:unit": "vitest",
+    "test:unit": "dotenv -e .env -- vitest",
     "test:typecheck": "tsc --noEmit"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.449.0",
     "@hono/node-server": "^1.2.0",
     "@hono/zod-openapi": "^0.8.3",
     "@prisma/client": "^5.5.2",
+    "crypto-hash": "^3.0.0",
+    "file-type": "^18.7.0",
     "hono": "^3.9.2",
     "http-errors": "^2.0.0",
     "p-map": "^6.0.0",
@@ -40,6 +43,7 @@
     "@types/http-errors": "^2.0.4",
     "@types/node": "^20.9.0",
     "del-cli": "^5.1.0",
+    "dotenv-cli": "^7.3.0",
     "eslint": "^8.53.0",
     "husky": "^8.0.3",
     "lint-staged": "^15.0.2",

src/files.ts

@@ -1,6 +1,9 @@
 import { OpenAPIHono } from '@hono/zod-openapi'
+import { sha256 } from 'crypto-hash'
+import { fileTypeFromBuffer } from 'file-type'
 
 import * as routes from './generated/oai-routes'
+import * as storage from './lib/storage'
 import * as utils from './lib/utils'
 import { prisma } from './lib/db'
 
@@ -29,15 +32,20 @@ app.openapi(routes.createFile, async (c) => {
 
   const { file, purpose } = body
 
-  // TODO: process file and upload to blob store
-  // TODO: extract file type and infer name
-  // TODO: correct byte length
+  const fileAsBuffer = Buffer.from(file, 'binary')
+  const fileType = await fileTypeFromBuffer(fileAsBuffer)
+  const fileName = `${sha256(file)}${fileType?.ext ? `.${fileType.ext}` : ''}`
+  const contentType = fileType?.mime || 'application/octet-stream'
+
+  await storage.putObject(fileName, fileAsBuffer, {
+    ContentType: contentType
+  })
 
   const res = await prisma.file.create({
     data: {
-      bytes: file.length,
-      filename: 'TODO',
+      filename: fileName,
       status: 'uploaded',
+      bytes: fileAsBuffer.byteLength,
       purpose
     }
   })

src/lib/storage.test.ts

@@ -0,0 +1,26 @@
+import { describe, expect, it } from 'vitest'
+
+import * as storage from './storage'
+
+describe('Storage', () => {
+  it('putObject, getObject, deleteObject', async () => {
+    if (!process.env.ACCESS_KEY_ID) {
+      // TODO: ignore on CI
+      expect(true).toEqual(true)
+      return
+    }
+
+    await storage.putObject('test.txt', 'hello world', {
+      ContentType: 'text/plain'
+    })
+
+    const obj = await storage.getObject('test.txt')
+    expect(obj.ContentType).toEqual('text/plain')
+
+    const body = await obj.Body?.transformToString()
+    expect(body).toEqual('hello world')
+
+    const res = await storage.deleteObject('test.txt')
+    expect(res.$metadata.httpStatusCode).toEqual(204)
+  })
+})

src/lib/storage.ts

@@ -0,0 +1,67 @@
+import {
+  DeleteObjectCommand,
+  type DeleteObjectCommandInput,
+  GetObjectCommand,
+  type GetObjectCommandInput,
+  PutObjectCommand,
+  type PutObjectCommandInput,
+  S3Client
+} from '@aws-sdk/client-s3'
+
+// This storage client is designed to work with any S3-compatible storage provider.
+// For Cloudflare R2, see https://developers.cloudflare.com/r2/examples/aws/aws-sdk-js-v3/
+
+export const bucket = process.env.S3_BUCKET!
+
+export const S3 = new S3Client({
+  region: process.env.S3_REGION ?? 'auto',
+  endpoint: process.env.S3_ENDPOINT!,
+  credentials: {
+    accessKeyId: process.env.ACCESS_KEY_ID!,
+    secretAccessKey: process.env.SECRET_ACCESS_KEY!
+  }
+})
+
+// This ensures that buckets are created automatically if they don't exist on
+// Cloudflare R2. It won't affect other providers.
+// @see https://developers.cloudflare.com/r2/examples/aws/custom-header/
+S3.middlewareStack.add(
+  (next, _) => async (args) => {
+    const r = args.request as RequestInit
+    r.headers = {
+      'cf-create-bucket-if-missing': 'true',
+      ...r.headers
+    }
+
+    return next(args)
+  },
+  { step: 'build', name: 'customHeaders' }
+)
+
+export async function getObject(
+  key: string,
+  opts?: Omit<GetObjectCommandInput, 'Bucket' | 'Key'>
+) {
+  return S3.send(new GetObjectCommand({ Bucket: bucket, Key: key, ...opts }))
+}
+
+export async function putObject(
+  key: string,
+  value: PutObjectCommandInput['Body'],
+  opts?: Omit<PutObjectCommandInput, 'Bucket' | 'Key' | 'Body'>
+) {
+  return S3.send(
+    new PutObjectCommand({ Bucket: bucket, Key: key, Body: value, ...opts })
+  )
+}
+
+export async function deleteObject(
+  key: string,
+  opts?: Omit<DeleteObjectCommandInput, 'Bucket' | 'Key'>
+) {
+  return S3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key, ...opts }))
+}
+
+export function getS3ObjectUrl(key: string) {
+  return `${process.env.S3_ENDPOINT}/${bucket}/${key}`
+}