s3.go 34 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045
  1. package handler
  2. import (
  3. "encoding/xml"
  4. "fmt"
  5. "io"
  6. "log"
  7. "net/http"
  8. "strings"
  9. "time"
  10. "aws-sts-mock/internal/service"
  11. "aws-sts-mock/internal/storage"
  12. "aws-sts-mock/pkg/checksum"
  13. "aws-sts-mock/pkg/s3"
  14. "aws-sts-mock/pkg/sigv4"
  15. )
  16. // S3Handler handles S3 HTTP requests with user isolation
  17. type S3Handler struct {
  18. service *service.S3Service
  19. }
  20. // NewS3Handler creates a new S3 handler
  21. func NewS3Handler(service *service.S3Service) *S3Handler {
  22. return &S3Handler{
  23. service: service,
  24. }
  25. }
  26. // Handle routes S3 requests to appropriate handlers
  27. // Helper functions - add these at package level (before S3Handler struct)
  28. func isVirtualHostedStyle(host string) bool {
  29. // Virtual-hosted-style: bucketname.s3.domain.com
  30. // Path-style: s3.domain.com
  31. parts := strings.Split(host, ".")
  32. // If host starts with a bucket name (has more than 2 parts before domain)
  33. // and contains "s3", it's virtual-hosted-style
  34. return len(parts) > 2 && strings.Contains(host, "s3")
  35. }
  36. func extractBucketFromHost(host string) string {
  37. // Extract bucket name from virtual-hosted-style host
  38. // photosbucket.s3.alanyeung.co -> photosbucket
  39. // Remove port if present
  40. if idx := strings.Index(host, ":"); idx != -1 {
  41. host = host[:idx]
  42. }
  43. parts := strings.Split(host, ".")
  44. if len(parts) > 0 {
  45. return parts[0]
  46. }
  47. return ""
  48. }
  49. // Handle routes S3 requests to appropriate handlers
  50. func (h *S3Handler) Handle(w http.ResponseWriter, r *http.Request) {
  51. log.Printf("S3 Request: %s %s from %s", r.Method, r.URL.Path, r.RemoteAddr)
  52. log.Printf("Host header: %s", r.Host)
  53. log.Printf("RequestURI: %s", r.RequestURI)
  54. var bucketName, objectKey string
  55. // Determine if this is virtual-hosted-style or path-style
  56. if isVirtualHostedStyle(r.Host) {
  57. // Virtual-hosted-style: bucket from Host header, object from path
  58. bucketName = extractBucketFromHost(r.Host)
  59. objectKey = strings.TrimPrefix(r.URL.Path, "/")
  60. log.Printf("Virtual-hosted-style request: bucket=%s, key=%s", bucketName, objectKey)
  61. } else {
  62. // Path-style: bucket and object from path
  63. path := strings.TrimPrefix(r.URL.Path, "/")
  64. parts := strings.SplitN(path, "/", 2)
  65. // Root path - list buckets
  66. if path == "" {
  67. if r.Method == "GET" {
  68. h.handleListBuckets(w, r)
  69. } else {
  70. h.writeError(w, "MethodNotAllowed", "Method not allowed", "", http.StatusMethodNotAllowed)
  71. }
  72. return
  73. }
  74. bucketName = parts[0]
  75. if len(parts) > 1 {
  76. objectKey = parts[1]
  77. }
  78. log.Printf("Path-style request: bucket=%s, key=%s", bucketName, objectKey)
  79. }
  80. // Bucket operations (no object key)
  81. if objectKey == "" {
  82. h.handleBucketOperation(w, r, bucketName)
  83. return
  84. }
  85. // Object operations
  86. h.handleObjectOperation(w, r, bucketName, objectKey)
  87. }
  88. // getUserFromContext extracts user credentials from request context
  89. func (h *S3Handler) getUserFromContext(r *http.Request) (sigv4.AWSCredentials, error) {
  90. creds, ok := r.Context().Value(sigv4.CredentialsContextKey).(sigv4.AWSCredentials)
  91. if !ok {
  92. return sigv4.AWSCredentials{}, fmt.Errorf("failed to retrieve credentials")
  93. }
  94. return creds, nil
  95. }
  96. func (h *S3Handler) handleBucketOperation(w http.ResponseWriter, r *http.Request, bucketName string) {
  97. query := r.URL.Query()
  98. // Check for list multipart uploads
  99. if query.Has("uploads") && r.Method == "GET" {
  100. h.handleListMultipartUploads(w, r, bucketName)
  101. return
  102. }
  103. // Check for delete query parameter (for DeleteObjects)
  104. if r.Method == "POST" && query.Get("delete") != "" {
  105. h.handleDeleteObjects(w, r, bucketName)
  106. return
  107. }
  108. switch r.Method {
  109. case "PUT":
  110. h.handleCreateBucket(w, r, bucketName)
  111. case "GET":
  112. h.handleListObjects(w, r, bucketName)
  113. case "DELETE":
  114. h.handleDeleteBucket(w, r, bucketName)
  115. case "HEAD":
  116. h.handleHeadBucket(w, r, bucketName)
  117. default:
  118. h.writeError(w, "MethodNotAllowed", "Method not allowed", bucketName, http.StatusMethodNotAllowed)
  119. }
  120. }
  121. func (h *S3Handler) handleObjectOperation(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  122. query := r.URL.Query()
  123. // Check for multipart operations
  124. if query.Has("uploads") && r.Method == "POST" {
  125. h.handleCreateMultipartUpload(w, r, bucketName, objectKey)
  126. return
  127. }
  128. uploadID := query.Get("uploadId")
  129. if uploadID != "" {
  130. if r.Method == "PUT" && query.Has("partNumber") {
  131. h.handleUploadPart(w, r, bucketName, objectKey)
  132. return
  133. }
  134. if r.Method == "POST" {
  135. h.handleCompleteMultipartUpload(w, r, bucketName, objectKey)
  136. return
  137. }
  138. if r.Method == "DELETE" {
  139. h.handleAbortMultipartUpload(w, r, bucketName, objectKey)
  140. return
  141. }
  142. if r.Method == "GET" {
  143. h.handleListParts(w, r, bucketName, objectKey)
  144. return
  145. }
  146. }
  147. // Regular object operations
  148. switch r.Method {
  149. case "PUT":
  150. h.handlePutObject(w, r, bucketName, objectKey)
  151. case "GET":
  152. h.handleGetObject(w, r, bucketName, objectKey)
  153. case "DELETE":
  154. h.handleDeleteObject(w, r, bucketName, objectKey)
  155. case "HEAD":
  156. h.handleHeadObject(w, r, bucketName, objectKey)
  157. default:
  158. h.writeError(w, "MethodNotAllowed", "Method not allowed", objectKey, http.StatusMethodNotAllowed)
  159. }
  160. }
  161. func (h *S3Handler) handleListBuckets(w http.ResponseWriter, r *http.Request) {
  162. creds, err := h.getUserFromContext(r)
  163. if err != nil {
  164. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  165. return
  166. }
  167. result, err := h.service.ListBuckets(creds.AccountID, creds.AccessKeyID)
  168. if err != nil {
  169. log.Printf("Error listing buckets for user %s: %v", creds.AccountID, err)
  170. h.writeError(w, "InternalError", "Failed to list buckets", "", http.StatusInternalServerError)
  171. return
  172. }
  173. result.XMLName = xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListAllMyBucketsResult"}
  174. w.Header().Set("Content-Type", "application/xml")
  175. w.Header().Set("x-amz-request-id", generateRequestId())
  176. w.WriteHeader(http.StatusOK)
  177. encoder := xml.NewEncoder(w)
  178. encoder.Indent("", " ")
  179. if err := encoder.Encode(result); err != nil {
  180. log.Printf("Error encoding response: %v", err)
  181. }
  182. }
  183. func (h *S3Handler) handleCreateBucket(w http.ResponseWriter, r *http.Request, bucketName string) {
  184. creds, err := h.getUserFromContext(r)
  185. if err != nil {
  186. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  187. return
  188. }
  189. err = h.service.CreateBucket(creds.AccountID, bucketName)
  190. if err != nil {
  191. if err == storage.ErrBucketAlreadyExists {
  192. h.writeError(w, "BucketAlreadyExists", "The requested bucket name is not available", bucketName, http.StatusConflict)
  193. return
  194. }
  195. log.Printf("Failed to create bucket %s for user %s: %v", bucketName, creds.AccountID, err)
  196. h.writeError(w, "InternalError", "Failed to create bucket", bucketName, http.StatusInternalServerError)
  197. return
  198. }
  199. log.Printf("Created bucket: %s for user: %s", bucketName, creds.AccountID)
  200. w.Header().Set("Location", "/"+bucketName)
  201. w.Header().Set("x-amz-request-id", generateRequestId())
  202. w.WriteHeader(http.StatusOK)
  203. }
  204. func (h *S3Handler) handleListObjects(w http.ResponseWriter, r *http.Request, bucketName string) {
  205. creds, err := h.getUserFromContext(r)
  206. if err != nil {
  207. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  208. return
  209. }
  210. log.Printf("ListObjects request for bucket: %s, user: %s", bucketName, creds.AccountID)
  211. objects, err := h.service.ListObjects(creds.AccountID, bucketName)
  212. if err != nil {
  213. if err == storage.ErrBucketNotFound {
  214. h.writeError(w, "NoSuchBucket", "The specified bucket does not exist", bucketName, http.StatusNotFound)
  215. return
  216. }
  217. log.Printf("Failed to list objects in bucket %s for user %s: %v", bucketName, creds.AccountID, err)
  218. h.writeError(w, "InternalError", "Failed to list objects", bucketName, http.StatusInternalServerError)
  219. return
  220. }
  221. // Build object list XML
  222. var objectsXML []string
  223. for _, obj := range objects {
  224. objectsXML = append(objectsXML, fmt.Sprintf(` <Contents>
  225. <Key>%s</Key>
  226. <LastModified>%s</LastModified>
  227. <ETag>"%s"</ETag>
  228. <Size>%d</Size>
  229. <StorageClass>STANDARD</StorageClass>
  230. </Contents>`, obj.Key, obj.LastModified.UTC().Format(time.RFC3339), obj.ETag, obj.Size))
  231. }
  232. w.Header().Set("Content-Type", "application/xml")
  233. w.Header().Set("x-amz-request-id", generateRequestId())
  234. w.WriteHeader(http.StatusOK)
  235. response := fmt.Sprintf(`<?xml version="1.0" encoding="UTF-8"?>
  236. <ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
  237. <Name>%s</Name>
  238. <Prefix></Prefix>
  239. <Marker></Marker>
  240. <MaxKeys>1000</MaxKeys>
  241. <IsTruncated>false</IsTruncated>
  242. %s
  243. </ListBucketResult>`, bucketName, strings.Join(objectsXML, "\n"))
  244. w.Write([]byte(response))
  245. }
  246. func (h *S3Handler) handleDeleteBucket(w http.ResponseWriter, r *http.Request, bucketName string) {
  247. creds, err := h.getUserFromContext(r)
  248. if err != nil {
  249. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  250. return
  251. }
  252. // Check if bucket exists
  253. exists, err := h.service.BucketExists(creds.AccountID, bucketName)
  254. if err != nil {
  255. log.Printf("Error checking bucket existence %s for user %s: %v", bucketName, creds.AccountID, err)
  256. h.writeError(w, "InternalError", "Failed to check bucket", bucketName, http.StatusInternalServerError)
  257. return
  258. }
  259. if !exists {
  260. h.writeError(w, "NoSuchBucket", "The specified bucket does not exist", bucketName, http.StatusNotFound)
  261. return
  262. }
  263. // Check if bucket is empty
  264. isEmpty, err := h.service.BucketIsEmpty(creds.AccountID, bucketName)
  265. if err != nil {
  266. log.Printf("Error checking if bucket %s is empty for user %s: %v", bucketName, creds.AccountID, err)
  267. h.writeError(w, "InternalError", "Failed to check bucket contents", bucketName, http.StatusInternalServerError)
  268. return
  269. }
  270. if !isEmpty {
  271. h.writeError(w, "BucketNotEmpty", "The bucket you tried to delete is not empty", bucketName, http.StatusConflict)
  272. return
  273. }
  274. // Delete the bucket
  275. if err := h.service.DeleteBucket(creds.AccountID, bucketName); err != nil {
  276. log.Printf("Failed to delete bucket %s for user %s: %v", bucketName, creds.AccountID, err)
  277. if err == storage.ErrBucketNotFound {
  278. h.writeError(w, "NoSuchBucket", "The specified bucket does not exist", bucketName, http.StatusNotFound)
  279. return
  280. }
  281. h.writeError(w, "InternalError", "Failed to delete bucket", bucketName, http.StatusInternalServerError)
  282. return
  283. }
  284. log.Printf("Deleted bucket: %s for user: %s", bucketName, creds.AccountID)
  285. w.Header().Set("x-amz-request-id", generateRequestId())
  286. w.WriteHeader(http.StatusNoContent)
  287. }
  288. func (h *S3Handler) handleHeadBucket(w http.ResponseWriter, r *http.Request, bucketName string) {
  289. creds, err := h.getUserFromContext(r)
  290. if err != nil {
  291. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  292. return
  293. }
  294. exists, err := h.service.BucketExists(creds.AccountID, bucketName)
  295. if err != nil {
  296. w.WriteHeader(http.StatusInternalServerError)
  297. return
  298. }
  299. if !exists {
  300. w.WriteHeader(http.StatusNotFound)
  301. return
  302. }
  303. w.Header().Set("x-amz-request-id", generateRequestId())
  304. w.WriteHeader(http.StatusOK)
  305. }
  306. func (h *S3Handler) handlePutObject(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  307. creds, err := h.getUserFromContext(r)
  308. if err != nil {
  309. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  310. return
  311. }
  312. // Extract checksum headers
  313. checksumHeaders := extractChecksumHeaders(r)
  314. validator := checksum.NewValidator(checksumHeaders)
  315. // Validate and store the object
  316. result, data, err := validator.ValidateStream(r.Body)
  317. if err != nil {
  318. log.Printf("Failed to read object data %s/%s for user %s: %v", bucketName, objectKey, creds.AccountID, err)
  319. h.writeError(w, "InternalError", "Failed to read object data", objectKey, http.StatusInternalServerError)
  320. return
  321. }
  322. // Check if validation failed
  323. if !result.Valid {
  324. log.Printf("Checksum validation failed for %s/%s: %s", bucketName, objectKey, result.ErrorMessage)
  325. h.writeError(w, "InvalidDigest", result.ErrorMessage, objectKey, http.StatusBadRequest)
  326. return
  327. }
  328. // Store the object
  329. reader := strings.NewReader(string(data))
  330. err = h.service.PutObject(creds.AccountID, bucketName, objectKey, reader)
  331. if err != nil {
  332. if err == storage.ErrBucketNotFound {
  333. h.writeError(w, "NoSuchBucket", "The specified bucket does not exist", bucketName, http.StatusNotFound)
  334. return
  335. }
  336. log.Printf("Failed to put object %s/%s for user %s: %v", bucketName, objectKey, creds.AccountID, err)
  337. h.writeError(w, "InternalError", "Failed to create object", objectKey, http.StatusInternalServerError)
  338. return
  339. }
  340. log.Printf("Created object: %s/%s for user: %s (validated with %s)",
  341. bucketName, objectKey, creds.AccountID, result.ValidatedWith)
  342. // Set response headers including checksums
  343. responseHeaders := result.GetResponseHeaders()
  344. for key, value := range responseHeaders {
  345. w.Header().Set(key, value)
  346. }
  347. w.Header().Set("x-amz-request-id", generateRequestId())
  348. w.WriteHeader(http.StatusOK)
  349. }
  350. func (h *S3Handler) handleGetObject(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  351. creds, err := h.getUserFromContext(r)
  352. if err != nil {
  353. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  354. return
  355. }
  356. file, info, err := h.service.GetObject(creds.AccountID, bucketName, objectKey)
  357. if err != nil {
  358. if err == storage.ErrObjectNotFound {
  359. h.writeError(w, "NoSuchKey", "The specified key does not exist", objectKey, http.StatusNotFound)
  360. return
  361. }
  362. log.Printf("Failed to get object %s/%s for user %s: %v", bucketName, objectKey, creds.AccountID, err)
  363. h.writeError(w, "InternalError", "Failed to retrieve object", objectKey, http.StatusInternalServerError)
  364. return
  365. }
  366. defer file.Close()
  367. // Read file data to compute checksums
  368. data, err := io.ReadAll(file)
  369. if err != nil {
  370. log.Printf("Failed to read object data %s/%s: %v", bucketName, objectKey, err)
  371. h.writeError(w, "InternalError", "Failed to read object data", objectKey, http.StatusInternalServerError)
  372. return
  373. }
  374. // Compute checksums for the response
  375. validator := checksum.NewValidator(map[string]string{})
  376. result := validator.ComputeChecksums(data)
  377. log.Printf("Retrieved object: %s/%s for user: %s", bucketName, objectKey, creds.AccountID)
  378. // Set standard headers
  379. w.Header().Set("Content-Type", getContentType(objectKey))
  380. w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
  381. w.Header().Set("Last-Modified", info.LastModified.UTC().Format(http.TimeFormat))
  382. w.Header().Set("x-amz-request-id", generateRequestId())
  383. // Set checksum headers
  384. w.Header().Set("ETag", fmt.Sprintf(`"%s"`, result.MD5))
  385. w.Header().Set("x-amz-checksum-crc32", result.CRC32)
  386. w.Header().Set("x-amz-checksum-crc32c", result.CRC32C)
  387. w.Header().Set("x-amz-checksum-sha1", result.SHA1)
  388. w.Header().Set("x-amz-checksum-sha256", result.SHA256)
  389. w.WriteHeader(http.StatusOK)
  390. w.Write(data)
  391. }
  392. func (h *S3Handler) handleDeleteObject(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  393. creds, err := h.getUserFromContext(r)
  394. if err != nil {
  395. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  396. return
  397. }
  398. err = h.service.DeleteObject(creds.AccountID, bucketName, objectKey)
  399. if err != nil {
  400. log.Printf("Failed to delete object %s/%s for user %s: %v", bucketName, objectKey, creds.AccountID, err)
  401. }
  402. log.Printf("Deleted object: %s/%s for user: %s", bucketName, objectKey, creds.AccountID)
  403. w.Header().Set("x-amz-request-id", generateRequestId())
  404. w.WriteHeader(http.StatusNoContent)
  405. }
  406. func (h *S3Handler) handleHeadObject(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  407. creds, err := h.getUserFromContext(r)
  408. if err != nil {
  409. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  410. return
  411. }
  412. info, err := h.service.GetObjectInfo(creds.AccountID, bucketName, objectKey)
  413. if err != nil {
  414. if err == storage.ErrObjectNotFound {
  415. w.WriteHeader(http.StatusNotFound)
  416. return
  417. }
  418. w.WriteHeader(http.StatusInternalServerError)
  419. return
  420. }
  421. w.Header().Set("Content-Type", getContentType(objectKey))
  422. w.Header().Set("Content-Length", fmt.Sprintf("%d", info.Size))
  423. w.Header().Set("Last-Modified", info.LastModified.UTC().Format(http.TimeFormat))
  424. w.Header().Set("ETag", fmt.Sprintf("\"%s\"", info.ETag))
  425. w.Header().Set("x-amz-request-id", generateRequestId())
  426. w.WriteHeader(http.StatusOK)
  427. }
  428. func (h *S3Handler) writeError(w http.ResponseWriter, code, message, resource string, statusCode int) {
  429. errorResp := s3.S3Error{
  430. XMLName: xml.Name{Space: "", Local: "Error"},
  431. Code: code,
  432. Message: message,
  433. Resource: resource,
  434. RequestId: generateRequestId(),
  435. }
  436. w.Header().Set("Content-Type", "application/xml")
  437. w.Header().Set("x-amz-request-id", errorResp.RequestId)
  438. w.WriteHeader(statusCode)
  439. encoder := xml.NewEncoder(w)
  440. encoder.Indent("", " ")
  441. if err := encoder.Encode(errorResp); err != nil {
  442. log.Printf("Error encoding S3 error response: %v", err)
  443. }
  444. }
  445. // 123
  446. // Update handleDeleteBucket in handler/s3_handler.go
  447. // Update handleDeleteObjects to use the batch service method
  448. func (h *S3Handler) handleDeleteObjects(w http.ResponseWriter, r *http.Request, bucketName string) {
  449. creds, err := h.getUserFromContext(r)
  450. if err != nil {
  451. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  452. return
  453. }
  454. // Check if bucket exists
  455. exists, err := h.service.BucketExists(creds.AccountID, bucketName)
  456. if err != nil {
  457. log.Printf("Error checking bucket existence %s for user %s: %v", bucketName, creds.AccountID, err)
  458. h.writeError(w, "InternalError", "Failed to check bucket", bucketName, http.StatusInternalServerError)
  459. return
  460. }
  461. if !exists {
  462. h.writeError(w, "NoSuchBucket", "The specified bucket does not exist", bucketName, http.StatusNotFound)
  463. return
  464. }
  465. // Parse the delete request
  466. var deleteReq s3.DeleteObjectsRequest
  467. if err := xml.NewDecoder(r.Body).Decode(&deleteReq); err != nil {
  468. log.Printf("Error decoding delete objects request: %v", err)
  469. h.writeError(w, "MalformedXML", "The XML you provided was not well-formed", "", http.StatusBadRequest)
  470. return
  471. }
  472. // Validate request
  473. if len(deleteReq.Objects) == 0 {
  474. h.writeError(w, "MalformedXML", "No objects specified for deletion", "", http.StatusBadRequest)
  475. return
  476. }
  477. if len(deleteReq.Objects) > 1000 {
  478. h.writeError(w, "InvalidRequest", "You cannot delete more than 1000 objects in a single request", "", http.StatusBadRequest)
  479. return
  480. }
  481. // Extract keys
  482. keys := make([]string, len(deleteReq.Objects))
  483. for i, obj := range deleteReq.Objects {
  484. keys[i] = obj.Key
  485. }
  486. // Process deletions using service
  487. deletionErrors := h.service.DeleteObjects(creds.AccountID, bucketName, keys)
  488. // Build response
  489. response := s3.DeleteObjectsResponse{
  490. XMLName: xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "DeleteResult"},
  491. Deleted: []s3.DeletedObject{},
  492. Errors: []s3.DeleteError{},
  493. }
  494. for _, obj := range deleteReq.Objects {
  495. if err, hasError := deletionErrors[obj.Key]; hasError {
  496. log.Printf("Failed to delete object %s/%s for user %s: %v", bucketName, obj.Key, creds.AccountID, err)
  497. // Determine error code based on error type
  498. errorCode := "InternalError"
  499. errorMessage := "We encountered an internal error. Please try again."
  500. if err == storage.ErrObjectNotFound {
  501. // S3 doesn't return an error for missing objects in batch delete
  502. // Just add to deleted list
  503. if !deleteReq.Quiet {
  504. response.Deleted = append(response.Deleted, s3.DeletedObject{
  505. Key: obj.Key,
  506. })
  507. }
  508. continue
  509. }
  510. response.Errors = append(response.Errors, s3.DeleteError{
  511. Key: obj.Key,
  512. VersionId: obj.VersionId,
  513. Code: errorCode,
  514. Message: errorMessage,
  515. })
  516. } else {
  517. // Add to deleted list if not in quiet mode
  518. if !deleteReq.Quiet {
  519. response.Deleted = append(response.Deleted, s3.DeletedObject{
  520. Key: obj.Key,
  521. VersionId: obj.VersionId,
  522. })
  523. }
  524. }
  525. }
  526. log.Printf("Batch deleted %d objects (attempted: %d) from bucket %s for user: %s",
  527. len(deleteReq.Objects)-len(response.Errors), len(deleteReq.Objects), bucketName, creds.AccountID)
  528. // Write response
  529. w.Header().Set("Content-Type", "application/xml")
  530. w.Header().Set("x-amz-request-id", generateRequestId())
  531. w.WriteHeader(http.StatusOK)
  532. encoder := xml.NewEncoder(w)
  533. encoder.Indent("", " ")
  534. if err := encoder.Encode(response); err != nil {
  535. log.Printf("Error encoding delete objects response: %v", err)
  536. }
  537. }
  538. // Add these methods to S3Handler
  539. func (h *S3Handler) handleCreateMultipartUpload(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  540. creds, err := h.getUserFromContext(r)
  541. if err != nil {
  542. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  543. return
  544. }
  545. uploadID, err := h.service.CreateMultipartUpload(creds.AccountID, bucketName, objectKey)
  546. if err != nil {
  547. if err == storage.ErrBucketNotFound {
  548. h.writeError(w, "NoSuchBucket", "The specified bucket does not exist", bucketName, http.StatusNotFound)
  549. return
  550. }
  551. log.Printf("Failed to create multipart upload for %s/%s: %v", bucketName, objectKey, err)
  552. h.writeError(w, "InternalError", "Failed to initiate multipart upload", objectKey, http.StatusInternalServerError)
  553. return
  554. }
  555. log.Printf("Created multipart upload: %s for %s/%s, user: %s", uploadID, bucketName, objectKey, creds.AccountID)
  556. result := s3.InitiateMultipartUploadResult{
  557. XMLName: xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "InitiateMultipartUploadResult"},
  558. Bucket: bucketName,
  559. Key: objectKey,
  560. UploadId: uploadID,
  561. }
  562. w.Header().Set("Content-Type", "application/xml")
  563. w.Header().Set("x-amz-request-id", generateRequestId())
  564. w.WriteHeader(http.StatusOK)
  565. encoder := xml.NewEncoder(w)
  566. encoder.Indent("", " ")
  567. if err := encoder.Encode(result); err != nil {
  568. log.Printf("Error encoding response: %v", err)
  569. }
  570. }
  571. func (h *S3Handler) handleUploadPart(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  572. creds, err := h.getUserFromContext(r)
  573. if err != nil {
  574. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  575. return
  576. }
  577. // Get query parameters
  578. uploadID := r.URL.Query().Get("uploadId")
  579. partNumberStr := r.URL.Query().Get("partNumber")
  580. if uploadID == "" || partNumberStr == "" {
  581. h.writeError(w, "InvalidRequest", "Missing uploadId or partNumber", "", http.StatusBadRequest)
  582. return
  583. }
  584. partNumber := 0
  585. fmt.Sscanf(partNumberStr, "%d", &partNumber)
  586. // Verify upload belongs to user
  587. upload, err := h.service.GetMultipartUpload(uploadID)
  588. if err != nil {
  589. h.writeError(w, "NoSuchUpload", "The specified upload does not exist", uploadID, http.StatusNotFound)
  590. return
  591. }
  592. if upload.AccountID != creds.AccountID || upload.Bucket != bucketName || upload.Key != objectKey {
  593. h.writeError(w, "AccessDenied", "Access denied to this upload", uploadID, http.StatusForbidden)
  594. return
  595. }
  596. // Extract checksum headers and validate
  597. checksumHeaders := extractChecksumHeaders(r)
  598. validator := checksum.NewValidator(checksumHeaders)
  599. // Validate the part data
  600. result, data, err := validator.ValidateStream(r.Body)
  601. if err != nil {
  602. log.Printf("Failed to read part data for upload %s: %v", uploadID, err)
  603. h.writeError(w, "InternalError", "Failed to read part data", "", http.StatusInternalServerError)
  604. return
  605. }
  606. // Check if validation failed
  607. if !result.Valid {
  608. log.Printf("Checksum validation failed for part %d of upload %s: %s",
  609. partNumber, uploadID, result.ErrorMessage)
  610. h.writeError(w, "InvalidDigest", result.ErrorMessage, "", http.StatusBadRequest)
  611. return
  612. }
  613. // Upload the part
  614. reader := strings.NewReader(string(data))
  615. etag, err := h.service.UploadPart(uploadID, partNumber, reader)
  616. if err != nil {
  617. log.Printf("Failed to upload part %d for upload %s: %v", partNumber, uploadID, err)
  618. h.writeError(w, "InternalError", "Failed to upload part", "", http.StatusInternalServerError)
  619. return
  620. }
  621. log.Printf("Uploaded part %d for upload %s, user: %s (validated with %s)",
  622. partNumber, uploadID, creds.AccountID, result.ValidatedWith)
  623. // Set response headers including checksums
  624. w.Header().Set("ETag", etag)
  625. responseHeaders := result.GetResponseHeaders()
  626. for key, value := range responseHeaders {
  627. if key != "ETag" { // Don't override the ETag
  628. w.Header().Set(key, value)
  629. }
  630. }
  631. w.Header().Set("x-amz-request-id", generateRequestId())
  632. w.WriteHeader(http.StatusOK)
  633. }
  634. func (h *S3Handler) handleCompleteMultipartUpload(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  635. creds, err := h.getUserFromContext(r)
  636. if err != nil {
  637. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  638. return
  639. }
  640. uploadID := r.URL.Query().Get("uploadId")
  641. if uploadID == "" {
  642. h.writeError(w, "InvalidRequest", "Missing uploadId", "", http.StatusBadRequest)
  643. return
  644. }
  645. // Verify upload belongs to user
  646. upload, err := h.service.GetMultipartUpload(uploadID)
  647. if err != nil {
  648. h.writeError(w, "NoSuchUpload", "The specified upload does not exist", uploadID, http.StatusNotFound)
  649. return
  650. }
  651. if upload.AccountID != creds.AccountID || upload.Bucket != bucketName || upload.Key != objectKey {
  652. h.writeError(w, "AccessDenied", "Access denied to this upload", uploadID, http.StatusForbidden)
  653. return
  654. }
  655. // Parse request
  656. var req s3.CompleteMultipartUploadRequest
  657. if err := xml.NewDecoder(r.Body).Decode(&req); err != nil {
  658. log.Printf("Error decoding complete multipart upload request: %v", err)
  659. h.writeError(w, "MalformedXML", "The XML you provided was not well-formed", "", http.StatusBadRequest)
  660. return
  661. }
  662. // Extract part numbers
  663. parts := make([]int, len(req.Parts))
  664. for i, part := range req.Parts {
  665. parts[i] = part.PartNumber
  666. }
  667. // Complete the upload
  668. if err := h.service.CompleteMultipartUpload(uploadID, parts); err != nil {
  669. log.Printf("Failed to complete multipart upload %s: %v", uploadID, err)
  670. h.writeError(w, "InternalError", "Failed to complete multipart upload", "", http.StatusInternalServerError)
  671. return
  672. }
  673. log.Printf("Completed multipart upload: %s for %s/%s, user: %s", uploadID, bucketName, objectKey, creds.AccountID)
  674. result := s3.CompleteMultipartUploadResult{
  675. XMLName: xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "CompleteMultipartUploadResult"},
  676. Location: fmt.Sprintf("/%s/%s", bucketName, objectKey),
  677. Bucket: bucketName,
  678. Key: objectKey,
  679. ETag: fmt.Sprintf("\"%d\"", time.Now().Unix()),
  680. }
  681. w.Header().Set("Content-Type", "application/xml")
  682. w.Header().Set("x-amz-request-id", generateRequestId())
  683. w.WriteHeader(http.StatusOK)
  684. encoder := xml.NewEncoder(w)
  685. encoder.Indent("", " ")
  686. if err := encoder.Encode(result); err != nil {
  687. log.Printf("Error encoding response: %v", err)
  688. }
  689. }
  690. func (h *S3Handler) handleAbortMultipartUpload(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  691. creds, err := h.getUserFromContext(r)
  692. if err != nil {
  693. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  694. return
  695. }
  696. uploadID := r.URL.Query().Get("uploadId")
  697. if uploadID == "" {
  698. h.writeError(w, "InvalidRequest", "Missing uploadId", "", http.StatusBadRequest)
  699. return
  700. }
  701. // Verify upload belongs to user
  702. upload, err := h.service.GetMultipartUpload(uploadID)
  703. if err != nil {
  704. h.writeError(w, "NoSuchUpload", "The specified upload does not exist", uploadID, http.StatusNotFound)
  705. return
  706. }
  707. if upload.AccountID != creds.AccountID || upload.Bucket != bucketName || upload.Key != objectKey {
  708. h.writeError(w, "AccessDenied", "Access denied to this upload", uploadID, http.StatusForbidden)
  709. return
  710. }
  711. // Abort the upload
  712. if err := h.service.AbortMultipartUpload(uploadID); err != nil {
  713. log.Printf("Failed to abort multipart upload %s: %v", uploadID, err)
  714. h.writeError(w, "InternalError", "Failed to abort multipart upload", "", http.StatusInternalServerError)
  715. return
  716. }
  717. log.Printf("Aborted multipart upload: %s for %s/%s, user: %s", uploadID, bucketName, objectKey, creds.AccountID)
  718. w.Header().Set("x-amz-request-id", generateRequestId())
  719. w.WriteHeader(http.StatusNoContent)
  720. }
  721. func (h *S3Handler) handleListParts(w http.ResponseWriter, r *http.Request, bucketName, objectKey string) {
  722. creds, err := h.getUserFromContext(r)
  723. if err != nil {
  724. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  725. return
  726. }
  727. uploadID := r.URL.Query().Get("uploadId")
  728. if uploadID == "" {
  729. h.writeError(w, "InvalidRequest", "Missing uploadId", "", http.StatusBadRequest)
  730. return
  731. }
  732. // Verify upload belongs to user
  733. upload, err := h.service.GetMultipartUpload(uploadID)
  734. if err != nil {
  735. h.writeError(w, "NoSuchUpload", "The specified upload does not exist", uploadID, http.StatusNotFound)
  736. return
  737. }
  738. if upload.AccountID != creds.AccountID || upload.Bucket != bucketName || upload.Key != objectKey {
  739. h.writeError(w, "AccessDenied", "Access denied to this upload", uploadID, http.StatusForbidden)
  740. return
  741. }
  742. // Parse query parameters
  743. maxParts := 1000
  744. partNumberMarker := 0
  745. if maxPartsStr := r.URL.Query().Get("max-parts"); maxPartsStr != "" {
  746. fmt.Sscanf(maxPartsStr, "%d", &maxParts)
  747. }
  748. if markerStr := r.URL.Query().Get("part-number-marker"); markerStr != "" {
  749. fmt.Sscanf(markerStr, "%d", &partNumberMarker)
  750. }
  751. // List parts
  752. parts, nextMarker, isTruncated, err := h.service.ListParts(uploadID, maxParts, partNumberMarker)
  753. if err != nil {
  754. log.Printf("Failed to list parts for upload %s: %v", uploadID, err)
  755. h.writeError(w, "InternalError", "Failed to list parts", "", http.StatusInternalServerError)
  756. return
  757. }
  758. // Build response
  759. s3Parts := make([]s3.Part, len(parts))
  760. for i, part := range parts {
  761. s3Parts[i] = s3.Part{
  762. PartNumber: part.PartNumber,
  763. LastModified: part.LastModified.UTC().Format(time.RFC3339),
  764. ETag: part.ETag,
  765. Size: part.Size,
  766. }
  767. }
  768. result := s3.ListPartsResult{
  769. XMLName: xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListPartsResult"},
  770. Bucket: bucketName,
  771. Key: objectKey,
  772. UploadId: uploadID,
  773. Initiator: s3.Initiator{ID: creds.AccountID, DisplayName: creds.AccessKeyID},
  774. Owner: s3.Owner{ID: creds.AccountID, DisplayName: creds.AccessKeyID},
  775. StorageClass: "STANDARD",
  776. PartNumberMarker: partNumberMarker,
  777. NextPartNumberMarker: nextMarker,
  778. MaxParts: maxParts,
  779. IsTruncated: isTruncated,
  780. Parts: s3Parts,
  781. }
  782. w.Header().Set("Content-Type", "application/xml")
  783. w.Header().Set("x-amz-request-id", generateRequestId())
  784. w.WriteHeader(http.StatusOK)
  785. encoder := xml.NewEncoder(w)
  786. encoder.Indent("", " ")
  787. if err := encoder.Encode(result); err != nil {
  788. log.Printf("Error encoding response: %v", err)
  789. }
  790. }
  791. func (h *S3Handler) handleListMultipartUploads(w http.ResponseWriter, r *http.Request, bucketName string) {
  792. creds, err := h.getUserFromContext(r)
  793. if err != nil {
  794. h.writeError(w, "AccessDenied", "Failed to retrieve credentials", "", http.StatusForbidden)
  795. return
  796. }
  797. // Parse query parameters
  798. maxUploads := 1000
  799. keyMarker := r.URL.Query().Get("key-marker")
  800. uploadIDMarker := r.URL.Query().Get("upload-id-marker")
  801. prefix := r.URL.Query().Get("prefix")
  802. if maxUploadsStr := r.URL.Query().Get("max-uploads"); maxUploadsStr != "" {
  803. fmt.Sscanf(maxUploadsStr, "%d", &maxUploads)
  804. }
  805. // List uploads
  806. uploads, nextKeyMarker, nextUploadIDMarker, isTruncated, err := h.service.ListMultipartUploads(
  807. creds.AccountID, bucketName, maxUploads, keyMarker, uploadIDMarker,
  808. )
  809. if err != nil {
  810. if err == storage.ErrBucketNotFound {
  811. h.writeError(w, "NoSuchBucket", "The specified bucket does not exist", bucketName, http.StatusNotFound)
  812. return
  813. }
  814. log.Printf("Failed to list multipart uploads for bucket %s: %v", bucketName, err)
  815. h.writeError(w, "InternalError", "Failed to list uploads", "", http.StatusInternalServerError)
  816. return
  817. }
  818. // Filter by prefix if provided
  819. if prefix != "" {
  820. filtered := []storage.MultipartUploadInfo{}
  821. for _, upload := range uploads {
  822. if len(upload.Key) >= len(prefix) && upload.Key[:len(prefix)] == prefix {
  823. filtered = append(filtered, upload)
  824. }
  825. }
  826. uploads = filtered
  827. }
  828. // Build response
  829. s3Uploads := make([]s3.MultipartUpload, len(uploads))
  830. for i, upload := range uploads {
  831. s3Uploads[i] = s3.MultipartUpload{
  832. Key: upload.Key,
  833. UploadId: upload.UploadID,
  834. Initiator: s3.Initiator{ID: creds.AccountID, DisplayName: creds.AccessKeyID},
  835. Owner: s3.Owner{ID: creds.AccountID, DisplayName: creds.AccessKeyID},
  836. StorageClass: "STANDARD",
  837. Initiated: upload.Initiated.UTC().Format(time.RFC3339),
  838. }
  839. }
  840. result := s3.ListMultipartUploadsResult{
  841. XMLName: xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListMultipartUploadsResult"},
  842. Bucket: bucketName,
  843. KeyMarker: keyMarker,
  844. UploadIdMarker: uploadIDMarker,
  845. NextKeyMarker: nextKeyMarker,
  846. NextUploadIdMarker: nextUploadIDMarker,
  847. MaxUploads: maxUploads,
  848. IsTruncated: isTruncated,
  849. Uploads: s3Uploads,
  850. Prefix: prefix,
  851. }
  852. w.Header().Set("Content-Type", "application/xml")
  853. w.Header().Set("x-amz-request-id", generateRequestId())
  854. w.WriteHeader(http.StatusOK)
  855. encoder := xml.NewEncoder(w)
  856. encoder.Indent("", " ")
  857. if err := encoder.Encode(result); err != nil {
  858. log.Printf("Error encoding response: %v", err)
  859. }
  860. }
  861. func extractChecksumHeaders(r *http.Request) map[string]string {
  862. headers := make(map[string]string)
  863. // Extract all checksum-related headers
  864. if val := r.Header.Get("Content-MD5"); val != "" {
  865. headers["Content-MD5"] = val
  866. }
  867. if val := r.Header.Get("x-amz-sdk-checksum-algorithm"); val != "" {
  868. headers["x-amz-sdk-checksum-algorithm"] = val
  869. }
  870. if val := r.Header.Get("x-amz-checksum-crc32"); val != "" {
  871. headers["x-amz-checksum-crc32"] = val
  872. }
  873. if val := r.Header.Get("x-amz-checksum-crc32c"); val != "" {
  874. headers["x-amz-checksum-crc32c"] = val
  875. }
  876. if val := r.Header.Get("x-amz-checksum-crc64nvme"); val != "" {
  877. headers["x-amz-checksum-crc64nvme"] = val
  878. }
  879. if val := r.Header.Get("x-amz-checksum-sha1"); val != "" {
  880. headers["x-amz-checksum-sha1"] = val
  881. }
  882. if val := r.Header.Get("x-amz-checksum-sha256"); val != "" {
  883. headers["x-amz-checksum-sha256"] = val
  884. }
  885. return headers
  886. }