mirror of
				https://codeberg.org/forgejo/forgejo.git
				synced 2025-10-31 14:31:02 +00:00 
			
		
		
		
	* Add a storage layer for attachments * Fix some bug * fix test * Fix copyright head and lint * Fix bug * Add setting for minio and flags for migrate-storage * Add documents * fix lint * Add test for minio store type on attachments * fix test * fix test * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Add warning when storage migrated successfully * Fix drone * fix test * rebase * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * remove log on xorm * Fi download bug * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * refactor the codes * add trace * Fix test * Add URL function to serve attachments directly from S3/Minio * Add ability to enable/disable redirection in attachment configuration * Fix typo * Add a storage layer for attachments * Add setting for minio and flags for migrate-storage * fix lint * Add test for minio store type on attachments * Apply suggestions from code review Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> * Fix drone * fix test * Fix test * display the error on console * Move minio test to amd64 since minio docker don't support arm64 * don't change unrelated files * Fix lint * Fix build * update go.mod and go.sum * Use github.com/minio/minio-go/v6 * Remove unused function * Upgrade minio to v7 and some other improvements * fix lint * Fix go mod Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com> Co-authored-by: Tyler <tystuyfzand@gmail.com>
		
			
				
	
	
		
			255 lines
		
	
	
	
		
			7.7 KiB
		
	
	
	
		
			Go
		
	
	
	
		
			Vendored
		
	
	
	
			
		
		
	
	
			255 lines
		
	
	
	
		
			7.7 KiB
		
	
	
	
		
			Go
		
	
	
	
		
			Vendored
		
	
	
	
| /*
 | |
|  * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 | |
|  * Copyright 2017-2020 MinIO, Inc.
 | |
|  *
 | |
|  * Licensed under the Apache License, Version 2.0 (the "License");
 | |
|  * you may not use this file except in compliance with the License.
 | |
|  * You may obtain a copy of the License at
 | |
|  *
 | |
|  *     http://www.apache.org/licenses/LICENSE-2.0
 | |
|  *
 | |
|  * Unless required by applicable law or agreed to in writing, software
 | |
|  * distributed under the License is distributed on an "AS IS" BASIS,
 | |
|  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 | |
|  * See the License for the specific language governing permissions and
 | |
|  * limitations under the License.
 | |
|  */
 | |
| 
 | |
| package minio
 | |
| 
 | |
| import (
 | |
| 	"bufio"
 | |
| 	"bytes"
 | |
| 	"context"
 | |
| 	"encoding/xml"
 | |
| 	"net/http"
 | |
| 	"net/url"
 | |
| 	"time"
 | |
| 
 | |
| 	jsoniter "github.com/json-iterator/go"
 | |
| 	"github.com/minio/minio-go/v7/pkg/notification"
 | |
| 	"github.com/minio/minio-go/v7/pkg/s3utils"
 | |
| )
 | |
| 
 | |
| // SetBucketNotification saves a new bucket notification with a context to control cancellations and timeouts.
 | |
| func (c Client) SetBucketNotification(ctx context.Context, bucketName string, config notification.Configuration) error {
 | |
| 	// Input validation.
 | |
| 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 | |
| 		return err
 | |
| 	}
 | |
| 
 | |
| 	// Get resources properly escaped and lined up before
 | |
| 	// using them in http request.
 | |
| 	urlValues := make(url.Values)
 | |
| 	urlValues.Set("notification", "")
 | |
| 
 | |
| 	notifBytes, err := xml.Marshal(&config)
 | |
| 	if err != nil {
 | |
| 		return err
 | |
| 	}
 | |
| 
 | |
| 	notifBuffer := bytes.NewReader(notifBytes)
 | |
| 	reqMetadata := requestMetadata{
 | |
| 		bucketName:       bucketName,
 | |
| 		queryValues:      urlValues,
 | |
| 		contentBody:      notifBuffer,
 | |
| 		contentLength:    int64(len(notifBytes)),
 | |
| 		contentMD5Base64: sumMD5Base64(notifBytes),
 | |
| 		contentSHA256Hex: sum256Hex(notifBytes),
 | |
| 	}
 | |
| 
 | |
| 	// Execute PUT to upload a new bucket notification.
 | |
| 	resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
 | |
| 	defer closeResponse(resp)
 | |
| 	if err != nil {
 | |
| 		return err
 | |
| 	}
 | |
| 	if resp != nil {
 | |
| 		if resp.StatusCode != http.StatusOK {
 | |
| 			return httpRespToErrorResponse(resp, bucketName, "")
 | |
| 		}
 | |
| 	}
 | |
| 	return nil
 | |
| }
 | |
| 
 | |
| // RemoveAllBucketNotification - Remove bucket notification clears all previously specified config
 | |
| func (c Client) RemoveAllBucketNotification(ctx context.Context, bucketName string) error {
 | |
| 	return c.SetBucketNotification(ctx, bucketName, notification.Configuration{})
 | |
| }
 | |
| 
 | |
| // GetBucketNotification returns current bucket notification configuration
 | |
| func (c Client) GetBucketNotification(ctx context.Context, bucketName string) (bucketNotification notification.Configuration, err error) {
 | |
| 	// Input validation.
 | |
| 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 | |
| 		return notification.Configuration{}, err
 | |
| 	}
 | |
| 	return c.getBucketNotification(ctx, bucketName)
 | |
| }
 | |
| 
 | |
| // Request server for notification rules.
 | |
| func (c Client) getBucketNotification(ctx context.Context, bucketName string) (notification.Configuration, error) {
 | |
| 	urlValues := make(url.Values)
 | |
| 	urlValues.Set("notification", "")
 | |
| 
 | |
| 	// Execute GET on bucket to list objects.
 | |
| 	resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
 | |
| 		bucketName:       bucketName,
 | |
| 		queryValues:      urlValues,
 | |
| 		contentSHA256Hex: emptySHA256Hex,
 | |
| 	})
 | |
| 
 | |
| 	defer closeResponse(resp)
 | |
| 	if err != nil {
 | |
| 		return notification.Configuration{}, err
 | |
| 	}
 | |
| 	return processBucketNotificationResponse(bucketName, resp)
 | |
| 
 | |
| }
 | |
| 
 | |
| // processes the GetNotification http response from the server.
 | |
| func processBucketNotificationResponse(bucketName string, resp *http.Response) (notification.Configuration, error) {
 | |
| 	if resp.StatusCode != http.StatusOK {
 | |
| 		errResponse := httpRespToErrorResponse(resp, bucketName, "")
 | |
| 		return notification.Configuration{}, errResponse
 | |
| 	}
 | |
| 	var bucketNotification notification.Configuration
 | |
| 	err := xmlDecoder(resp.Body, &bucketNotification)
 | |
| 	if err != nil {
 | |
| 		return notification.Configuration{}, err
 | |
| 	}
 | |
| 	return bucketNotification, nil
 | |
| }
 | |
| 
 | |
| // ListenNotification listen for all events, this is a MinIO specific API
 | |
| func (c Client) ListenNotification(ctx context.Context, prefix, suffix string, events []string) <-chan notification.Info {
 | |
| 	return c.ListenBucketNotification(ctx, "", prefix, suffix, events)
 | |
| }
 | |
| 
 | |
| // ListenBucketNotification listen for bucket events, this is a MinIO specific API
 | |
| func (c Client) ListenBucketNotification(ctx context.Context, bucketName, prefix, suffix string, events []string) <-chan notification.Info {
 | |
| 	notificationInfoCh := make(chan notification.Info, 1)
 | |
| 	const notificationCapacity = 4 * 1024 * 1024
 | |
| 	notificationEventBuffer := make([]byte, notificationCapacity)
 | |
| 	// Only success, start a routine to start reading line by line.
 | |
| 	go func(notificationInfoCh chan<- notification.Info) {
 | |
| 		defer close(notificationInfoCh)
 | |
| 
 | |
| 		// Validate the bucket name.
 | |
| 		if bucketName != "" {
 | |
| 			if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 | |
| 				select {
 | |
| 				case notificationInfoCh <- notification.Info{
 | |
| 					Err: err,
 | |
| 				}:
 | |
| 				case <-ctx.Done():
 | |
| 				}
 | |
| 				return
 | |
| 			}
 | |
| 		}
 | |
| 
 | |
| 		// Check ARN partition to verify if listening bucket is supported
 | |
| 		if s3utils.IsAmazonEndpoint(*c.endpointURL) || s3utils.IsGoogleEndpoint(*c.endpointURL) {
 | |
| 			select {
 | |
| 			case notificationInfoCh <- notification.Info{
 | |
| 				Err: errAPINotSupported("Listening for bucket notification is specific only to `minio` server endpoints"),
 | |
| 			}:
 | |
| 			case <-ctx.Done():
 | |
| 			}
 | |
| 			return
 | |
| 		}
 | |
| 
 | |
| 		// Continuously run and listen on bucket notification.
 | |
| 		// Create a done channel to control 'ListObjects' go routine.
 | |
| 		retryDoneCh := make(chan struct{}, 1)
 | |
| 
 | |
| 		// Indicate to our routine to exit cleanly upon return.
 | |
| 		defer close(retryDoneCh)
 | |
| 
 | |
| 		// Prepare urlValues to pass into the request on every loop
 | |
| 		urlValues := make(url.Values)
 | |
| 		urlValues.Set("prefix", prefix)
 | |
| 		urlValues.Set("suffix", suffix)
 | |
| 		urlValues["events"] = events
 | |
| 
 | |
| 		// Wait on the jitter retry loop.
 | |
| 		for range c.newRetryTimerContinous(time.Second, time.Second*30, MaxJitter, retryDoneCh) {
 | |
| 			// Execute GET on bucket to list objects.
 | |
| 			resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
 | |
| 				bucketName:       bucketName,
 | |
| 				queryValues:      urlValues,
 | |
| 				contentSHA256Hex: emptySHA256Hex,
 | |
| 			})
 | |
| 			if err != nil {
 | |
| 				select {
 | |
| 				case notificationInfoCh <- notification.Info{
 | |
| 					Err: err,
 | |
| 				}:
 | |
| 				case <-ctx.Done():
 | |
| 				}
 | |
| 				return
 | |
| 			}
 | |
| 
 | |
| 			// Validate http response, upon error return quickly.
 | |
| 			if resp.StatusCode != http.StatusOK {
 | |
| 				errResponse := httpRespToErrorResponse(resp, bucketName, "")
 | |
| 				select {
 | |
| 				case notificationInfoCh <- notification.Info{
 | |
| 					Err: errResponse,
 | |
| 				}:
 | |
| 				case <-ctx.Done():
 | |
| 				}
 | |
| 				return
 | |
| 			}
 | |
| 
 | |
| 			// Initialize a new bufio scanner, to read line by line.
 | |
| 			bio := bufio.NewScanner(resp.Body)
 | |
| 
 | |
| 			// Use a higher buffer to support unexpected
 | |
| 			// caching done by proxies
 | |
| 			bio.Buffer(notificationEventBuffer, notificationCapacity)
 | |
| 			var json = jsoniter.ConfigCompatibleWithStandardLibrary
 | |
| 
 | |
| 			// Unmarshal each line, returns marshaled values.
 | |
| 			for bio.Scan() {
 | |
| 				var notificationInfo notification.Info
 | |
| 				if err = json.Unmarshal(bio.Bytes(), ¬ificationInfo); err != nil {
 | |
| 					// Unexpected error during json unmarshal, send
 | |
| 					// the error to caller for actionable as needed.
 | |
| 					select {
 | |
| 					case notificationInfoCh <- notification.Info{
 | |
| 						Err: err,
 | |
| 					}:
 | |
| 					case <-ctx.Done():
 | |
| 						return
 | |
| 					}
 | |
| 					closeResponse(resp)
 | |
| 					continue
 | |
| 				}
 | |
| 				// Send notificationInfo
 | |
| 				select {
 | |
| 				case notificationInfoCh <- notificationInfo:
 | |
| 				case <-ctx.Done():
 | |
| 					closeResponse(resp)
 | |
| 					return
 | |
| 				}
 | |
| 			}
 | |
| 
 | |
| 			if err = bio.Err(); err != nil {
 | |
| 				select {
 | |
| 				case notificationInfoCh <- notification.Info{
 | |
| 					Err: err,
 | |
| 				}:
 | |
| 				case <-ctx.Done():
 | |
| 					return
 | |
| 				}
 | |
| 			}
 | |
| 
 | |
| 			// Close current connection before looping further.
 | |
| 			closeResponse(resp)
 | |
| 
 | |
| 		}
 | |
| 	}(notificationInfoCh)
 | |
| 
 | |
| 	// Returns the notification info channel, for caller to start reading from.
 | |
| 	return notificationInfoCh
 | |
| }
 |