|
@@ -0,0 +1,425 @@
|
|
|
|
+package checksum
|
|
|
|
+
|
|
|
|
+import (
|
|
|
|
+ "crypto/md5"
|
|
|
|
+ "crypto/sha1"
|
|
|
|
+ "crypto/sha256"
|
|
|
|
+ "encoding/base64"
|
|
|
|
+ "fmt"
|
|
|
|
+ "hash"
|
|
|
|
+ "hash/crc32"
|
|
|
|
+ "hash/crc64"
|
|
|
|
+ "io"
|
|
|
|
+)
|
|
|
|
+
|
|
|
|
+// ChecksumAlgorithm represents supported checksum algorithms
|
|
|
|
+type ChecksumAlgorithm string
|
|
|
|
+
|
|
|
|
+const (
|
|
|
|
+ AlgorithmMD5 ChecksumAlgorithm = "MD5"
|
|
|
|
+ AlgorithmCRC32 ChecksumAlgorithm = "CRC32"
|
|
|
|
+ AlgorithmCRC32C ChecksumAlgorithm = "CRC32C"
|
|
|
|
+ AlgorithmCRC64NVME ChecksumAlgorithm = "CRC64NVME"
|
|
|
|
+ AlgorithmSHA1 ChecksumAlgorithm = "SHA1"
|
|
|
|
+ AlgorithmSHA256 ChecksumAlgorithm = "SHA256"
|
|
|
|
+)
|
|
|
|
+
|
|
|
|
+// ChecksumValidator validates data against provided checksums
|
|
|
|
+type ChecksumValidator struct {
|
|
|
|
+ // Algorithm specified in x-amz-sdk-checksum-algorithm header
|
|
|
|
+ Algorithm ChecksumAlgorithm
|
|
|
|
+
|
|
|
|
+ // Expected checksums from headers (base64 encoded)
|
|
|
|
+ ExpectedMD5 string // Content-MD5 header
|
|
|
|
+ ExpectedCRC32 string // x-amz-checksum-crc32 header
|
|
|
|
+ ExpectedCRC32C string // x-amz-checksum-crc32c header
|
|
|
|
+ ExpectedCRC64NVME string // x-amz-checksum-crc64nvme header
|
|
|
|
+ ExpectedSHA1 string // x-amz-checksum-sha1 header
|
|
|
|
+ ExpectedSHA256 string // x-amz-checksum-sha256 header
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ChecksumResult contains computed checksums and validation results
|
|
|
|
+type ChecksumResult struct {
|
|
|
|
+ MD5 string // base64 encoded
|
|
|
|
+ CRC32 string // base64 encoded
|
|
|
|
+ CRC32C string // base64 encoded
|
|
|
|
+ CRC64NVME string // base64 encoded
|
|
|
|
+ SHA1 string // base64 encoded
|
|
|
|
+ SHA256 string // base64 encoded
|
|
|
|
+
|
|
|
|
+ // Validation results
|
|
|
|
+ Valid bool
|
|
|
|
+ ValidatedWith ChecksumAlgorithm
|
|
|
|
+ ErrorMessage string
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// NewValidator creates a new checksum validator from headers
|
|
|
|
+func NewValidator(headers map[string]string) *ChecksumValidator {
|
|
|
|
+ validator := &ChecksumValidator{
|
|
|
|
+ Algorithm: ChecksumAlgorithm(headers["x-amz-sdk-checksum-algorithm"]),
|
|
|
|
+ ExpectedMD5: headers["Content-MD5"],
|
|
|
|
+ ExpectedCRC32: headers["x-amz-checksum-crc32"],
|
|
|
|
+ ExpectedCRC32C: headers["x-amz-checksum-crc32c"],
|
|
|
|
+ ExpectedCRC64NVME: headers["x-amz-checksum-crc64nvme"],
|
|
|
|
+ ExpectedSHA1: headers["x-amz-checksum-sha1"],
|
|
|
|
+ ExpectedSHA256: headers["x-amz-checksum-sha256"],
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ // Normalize algorithm name
|
|
|
|
+ if validator.Algorithm == "" {
|
|
|
|
+ // If no algorithm specified but MD5 is present, use MD5
|
|
|
|
+ if validator.ExpectedMD5 != "" {
|
|
|
|
+ validator.Algorithm = AlgorithmMD5
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ return validator
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ValidateData validates data against expected checksums
|
|
|
|
+func (v *ChecksumValidator) ValidateData(data []byte) *ChecksumResult {
|
|
|
|
+ result := v.ComputeChecksums(data)
|
|
|
|
+
|
|
|
|
+ // Validate based on algorithm or any provided checksum
|
|
|
|
+ if v.Algorithm != "" {
|
|
|
|
+ result.ValidatedWith = v.Algorithm
|
|
|
|
+ result.Valid = v.validateAlgorithm(result, v.Algorithm)
|
|
|
|
+ if !result.Valid {
|
|
|
|
+ result.ErrorMessage = fmt.Sprintf("Checksum mismatch for algorithm %s", v.Algorithm)
|
|
|
|
+ }
|
|
|
|
+ } else {
|
|
|
|
+ // Try to validate with any available checksum
|
|
|
|
+ result.Valid = true // Assume valid if no checksums provided
|
|
|
|
+
|
|
|
|
+ if v.ExpectedMD5 != "" {
|
|
|
|
+ result.ValidatedWith = AlgorithmMD5
|
|
|
|
+ if !v.validateAlgorithm(result, AlgorithmMD5) {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ErrorMessage = "MD5 checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if v.ExpectedCRC32 != "" {
|
|
|
|
+ result.ValidatedWith = AlgorithmCRC32
|
|
|
|
+ if !v.validateAlgorithm(result, AlgorithmCRC32) {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ErrorMessage = "CRC32 checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if v.ExpectedCRC32C != "" {
|
|
|
|
+ result.ValidatedWith = AlgorithmCRC32C
|
|
|
|
+ if !v.validateAlgorithm(result, AlgorithmCRC32C) {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ErrorMessage = "CRC32C checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if v.ExpectedCRC64NVME != "" {
|
|
|
|
+ result.ValidatedWith = AlgorithmCRC64NVME
|
|
|
|
+ if !v.validateAlgorithm(result, AlgorithmCRC64NVME) {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ErrorMessage = "CRC64NVME checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if v.ExpectedSHA1 != "" {
|
|
|
|
+ result.ValidatedWith = AlgorithmSHA1
|
|
|
|
+ if !v.validateAlgorithm(result, AlgorithmSHA1) {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ErrorMessage = "SHA1 checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if v.ExpectedSHA256 != "" {
|
|
|
|
+ result.ValidatedWith = AlgorithmSHA256
|
|
|
|
+ if !v.validateAlgorithm(result, AlgorithmSHA256) {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ErrorMessage = "SHA256 checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ return result
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ValidateStream validates a stream of data against expected checksums
|
|
|
|
+func (v *ChecksumValidator) ValidateStream(reader io.Reader) (*ChecksumResult, []byte, error) {
|
|
|
|
+ // Read all data
|
|
|
|
+ data, err := io.ReadAll(reader)
|
|
|
|
+ if err != nil {
|
|
|
|
+ return nil, nil, fmt.Errorf("failed to read data: %w", err)
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ result := v.ValidateData(data)
|
|
|
|
+ return result, data, nil
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// validateAlgorithm validates a specific algorithm
|
|
|
|
+func (v *ChecksumValidator) validateAlgorithm(result *ChecksumResult, algorithm ChecksumAlgorithm) bool {
|
|
|
|
+ switch algorithm {
|
|
|
|
+ case AlgorithmMD5:
|
|
|
|
+ return v.ExpectedMD5 == "" || v.ExpectedMD5 == result.MD5
|
|
|
|
+ case AlgorithmCRC32:
|
|
|
|
+ return v.ExpectedCRC32 == "" || v.ExpectedCRC32 == result.CRC32
|
|
|
|
+ case AlgorithmCRC32C:
|
|
|
|
+ return v.ExpectedCRC32C == "" || v.ExpectedCRC32C == result.CRC32C
|
|
|
|
+ case AlgorithmCRC64NVME:
|
|
|
|
+ return v.ExpectedCRC64NVME == "" || v.ExpectedCRC64NVME == result.CRC64NVME
|
|
|
|
+ case AlgorithmSHA1:
|
|
|
|
+ return v.ExpectedSHA1 == "" || v.ExpectedSHA1 == result.SHA1
|
|
|
|
+ case AlgorithmSHA256:
|
|
|
|
+ return v.ExpectedSHA256 == "" || v.ExpectedSHA256 == result.SHA256
|
|
|
|
+ default:
|
|
|
|
+ return false
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ComputeChecksums computes all supported checksums for the data
|
|
|
|
+func (v *ChecksumValidator) ComputeChecksums(data []byte) *ChecksumResult {
|
|
|
|
+ result := &ChecksumResult{
|
|
|
|
+ Valid: true,
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ // Compute all checksums
|
|
|
|
+ result.MD5 = computeMD5(data)
|
|
|
|
+ result.CRC32 = computeCRC32(data)
|
|
|
|
+ result.CRC32C = computeCRC32C(data)
|
|
|
|
+ result.CRC64NVME = computeCRC64NVME(data)
|
|
|
|
+ result.SHA1 = computeSHA1(data)
|
|
|
|
+ result.SHA256 = computeSHA256(data)
|
|
|
|
+
|
|
|
|
+ return result
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// GetResponseHeaders returns the appropriate checksum headers for the response
|
|
|
|
+func (r *ChecksumResult) GetResponseHeaders() map[string]string {
|
|
|
|
+ headers := make(map[string]string)
|
|
|
|
+
|
|
|
|
+ // Always include ETag (MD5)
|
|
|
|
+ headers["ETag"] = fmt.Sprintf(`"%s"`, r.MD5)
|
|
|
|
+
|
|
|
|
+ // Include checksum headers based on what was validated
|
|
|
|
+ switch r.ValidatedWith {
|
|
|
|
+ case AlgorithmCRC32:
|
|
|
|
+ headers["x-amz-checksum-crc32"] = r.CRC32
|
|
|
|
+ headers["x-amz-checksum-type"] = "COMPOSITE" // For multipart uploads
|
|
|
|
+ case AlgorithmCRC32C:
|
|
|
|
+ headers["x-amz-checksum-crc32c"] = r.CRC32C
|
|
|
|
+ headers["x-amz-checksum-type"] = "COMPOSITE"
|
|
|
|
+ case AlgorithmCRC64NVME:
|
|
|
|
+ headers["x-amz-checksum-crc64nvme"] = r.CRC64NVME
|
|
|
|
+ headers["x-amz-checksum-type"] = "COMPOSITE"
|
|
|
|
+ case AlgorithmSHA1:
|
|
|
|
+ headers["x-amz-checksum-sha1"] = r.SHA1
|
|
|
|
+ headers["x-amz-checksum-type"] = "COMPOSITE"
|
|
|
|
+ case AlgorithmSHA256:
|
|
|
|
+ headers["x-amz-checksum-sha256"] = r.SHA256
|
|
|
|
+ headers["x-amz-checksum-type"] = "COMPOSITE"
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ return headers
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// Helper functions to compute individual checksums
|
|
|
|
+
|
|
|
|
+func computeMD5(data []byte) string {
|
|
|
|
+ h := md5.New()
|
|
|
|
+ h.Write(data)
|
|
|
|
+ return base64.StdEncoding.EncodeToString(h.Sum(nil))
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+func computeCRC32(data []byte) string {
|
|
|
|
+ h := crc32.NewIEEE()
|
|
|
|
+ h.Write(data)
|
|
|
|
+ sum := h.Sum32()
|
|
|
|
+ // Convert to 4-byte array
|
|
|
|
+ bytes := []byte{
|
|
|
|
+ byte(sum >> 24),
|
|
|
|
+ byte(sum >> 16),
|
|
|
|
+ byte(sum >> 8),
|
|
|
|
+ byte(sum),
|
|
|
|
+ }
|
|
|
|
+ return base64.StdEncoding.EncodeToString(bytes)
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+func computeCRC32C(data []byte) string {
|
|
|
|
+ // CRC32C uses Castagnoli polynomial
|
|
|
|
+ table := crc32.MakeTable(crc32.Castagnoli)
|
|
|
|
+ h := crc32.New(table)
|
|
|
|
+ h.Write(data)
|
|
|
|
+ sum := h.Sum32()
|
|
|
|
+ // Convert to 4-byte array
|
|
|
|
+ bytes := []byte{
|
|
|
|
+ byte(sum >> 24),
|
|
|
|
+ byte(sum >> 16),
|
|
|
|
+ byte(sum >> 8),
|
|
|
|
+ byte(sum),
|
|
|
|
+ }
|
|
|
|
+ return base64.StdEncoding.EncodeToString(bytes)
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+func computeCRC64NVME(data []byte) string {
|
|
|
|
+ // CRC64 with NVME polynomial
|
|
|
|
+ // Using ECMA polynomial as approximation (Go doesn't have NVME built-in)
|
|
|
|
+ table := crc64.MakeTable(crc64.ECMA)
|
|
|
|
+ h := crc64.New(table)
|
|
|
|
+ h.Write(data)
|
|
|
|
+ sum := h.Sum64()
|
|
|
|
+ // Convert to 8-byte array
|
|
|
|
+ bytes := []byte{
|
|
|
|
+ byte(sum >> 56),
|
|
|
|
+ byte(sum >> 48),
|
|
|
|
+ byte(sum >> 40),
|
|
|
|
+ byte(sum >> 32),
|
|
|
|
+ byte(sum >> 24),
|
|
|
|
+ byte(sum >> 16),
|
|
|
|
+ byte(sum >> 8),
|
|
|
|
+ byte(sum),
|
|
|
|
+ }
|
|
|
|
+ return base64.StdEncoding.EncodeToString(bytes)
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+func computeSHA1(data []byte) string {
|
|
|
|
+ h := sha1.New()
|
|
|
|
+ h.Write(data)
|
|
|
|
+ return base64.StdEncoding.EncodeToString(h.Sum(nil))
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+func computeSHA256(data []byte) string {
|
|
|
|
+ h := sha256.New()
|
|
|
|
+ h.Write(data)
|
|
|
|
+ return base64.StdEncoding.EncodeToString(h.Sum(nil))
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// StreamingValidator allows computing checksums while streaming data
|
|
|
|
+type StreamingValidator struct {
|
|
|
|
+ md5Hash hash.Hash
|
|
|
|
+ crc32Hash hash.Hash32
|
|
|
|
+ crc32cHash hash.Hash32
|
|
|
|
+ crc64nvmeHash hash.Hash64
|
|
|
|
+ sha1Hash hash.Hash
|
|
|
|
+ sha256Hash hash.Hash
|
|
|
|
+
|
|
|
|
+ bytesWritten int64
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// NewStreamingValidator creates a new streaming validator
|
|
|
|
+func NewStreamingValidator() *StreamingValidator {
|
|
|
|
+ return &StreamingValidator{
|
|
|
|
+ md5Hash: md5.New(),
|
|
|
|
+ crc32Hash: crc32.NewIEEE(),
|
|
|
|
+ crc32cHash: crc32.New(crc32.MakeTable(crc32.Castagnoli)),
|
|
|
|
+ crc64nvmeHash: crc64.New(crc64.MakeTable(crc64.ECMA)),
|
|
|
|
+ sha1Hash: sha1.New(),
|
|
|
|
+ sha256Hash: sha256.New(),
|
|
|
|
+ }
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// Write implements io.Writer
|
|
|
|
+func (sv *StreamingValidator) Write(p []byte) (n int, err error) {
|
|
|
|
+ sv.md5Hash.Write(p)
|
|
|
|
+ sv.crc32Hash.Write(p)
|
|
|
|
+ sv.crc32cHash.Write(p)
|
|
|
|
+ sv.crc64nvmeHash.Write(p)
|
|
|
|
+ sv.sha1Hash.Write(p)
|
|
|
|
+ sv.sha256Hash.Write(p)
|
|
|
|
+ sv.bytesWritten += int64(len(p))
|
|
|
|
+ return len(p), nil
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// GetResult returns the computed checksums
|
|
|
|
+func (sv *StreamingValidator) GetResult() *ChecksumResult {
|
|
|
|
+ result := &ChecksumResult{
|
|
|
|
+ MD5: base64.StdEncoding.EncodeToString(sv.md5Hash.Sum(nil)),
|
|
|
|
+ SHA1: base64.StdEncoding.EncodeToString(sv.sha1Hash.Sum(nil)),
|
|
|
|
+ SHA256: base64.StdEncoding.EncodeToString(sv.sha256Hash.Sum(nil)),
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ // CRC32
|
|
|
|
+ sum32 := sv.crc32Hash.Sum32()
|
|
|
|
+ result.CRC32 = base64.StdEncoding.EncodeToString([]byte{
|
|
|
|
+ byte(sum32 >> 24), byte(sum32 >> 16), byte(sum32 >> 8), byte(sum32),
|
|
|
|
+ })
|
|
|
|
+
|
|
|
|
+ // CRC32C
|
|
|
|
+ sum32c := sv.crc32cHash.Sum32()
|
|
|
|
+ result.CRC32C = base64.StdEncoding.EncodeToString([]byte{
|
|
|
|
+ byte(sum32c >> 24), byte(sum32c >> 16), byte(sum32c >> 8), byte(sum32c),
|
|
|
|
+ })
|
|
|
|
+
|
|
|
|
+ // CRC64NVME
|
|
|
|
+ sum64 := sv.crc64nvmeHash.Sum64()
|
|
|
|
+ result.CRC64NVME = base64.StdEncoding.EncodeToString([]byte{
|
|
|
|
+ byte(sum64 >> 56), byte(sum64 >> 48), byte(sum64 >> 40), byte(sum64 >> 32),
|
|
|
|
+ byte(sum64 >> 24), byte(sum64 >> 16), byte(sum64 >> 8), byte(sum64),
|
|
|
|
+ })
|
|
|
|
+
|
|
|
|
+ return result
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// BytesWritten returns the total bytes written
|
|
|
|
+func (sv *StreamingValidator) BytesWritten() int64 {
|
|
|
|
+ return sv.bytesWritten
|
|
|
|
+}
|
|
|
|
+
|
|
|
|
+// ValidateWithExpected validates the computed checksums against expected values
|
|
|
|
+func (sv *StreamingValidator) ValidateWithExpected(validator *ChecksumValidator) *ChecksumResult {
|
|
|
|
+ result := sv.GetResult()
|
|
|
|
+
|
|
|
|
+ // Check each expected checksum
|
|
|
|
+ if validator.ExpectedMD5 != "" && validator.ExpectedMD5 != result.MD5 {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ValidatedWith = AlgorithmMD5
|
|
|
|
+ result.ErrorMessage = "MD5 checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if validator.ExpectedCRC32 != "" && validator.ExpectedCRC32 != result.CRC32 {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ValidatedWith = AlgorithmCRC32
|
|
|
|
+ result.ErrorMessage = "CRC32 checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if validator.ExpectedCRC32C != "" && validator.ExpectedCRC32C != result.CRC32C {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ValidatedWith = AlgorithmCRC32C
|
|
|
|
+ result.ErrorMessage = "CRC32C checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if validator.ExpectedCRC64NVME != "" && validator.ExpectedCRC64NVME != result.CRC64NVME {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ValidatedWith = AlgorithmCRC64NVME
|
|
|
|
+ result.ErrorMessage = "CRC64NVME checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if validator.ExpectedSHA1 != "" && validator.ExpectedSHA1 != result.SHA1 {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ValidatedWith = AlgorithmSHA1
|
|
|
|
+ result.ErrorMessage = "SHA1 checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if validator.ExpectedSHA256 != "" && validator.ExpectedSHA256 != result.SHA256 {
|
|
|
|
+ result.Valid = false
|
|
|
|
+ result.ValidatedWith = AlgorithmSHA256
|
|
|
|
+ result.ErrorMessage = "SHA256 checksum mismatch"
|
|
|
|
+ return result
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ result.Valid = true
|
|
|
|
+ if validator.Algorithm != "" {
|
|
|
|
+ result.ValidatedWith = validator.Algorithm
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ return result
|
|
|
|
+}
|