invisi/gotenberg
1
0
Fork 0
mirror of https://github.com/gotenberg/gotenberg.git synced 2026-07-02 08:27:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add metrics system, move webhook feature to dedicated module (#372)

Browse Source
This commit is contained in:
Julien Neuhart
2021-10-19 18:35:24 +02:00
committed by GitHub
parent b839069af1
commit d4c47cecf2
39 changed files with 3359 additions and 1309 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/modules/api/api.go
+134 -133
View File
@@ -6,7 +6,6 @@ import (
"fmt"
"net/http"
"os"
"regexp"
"sort"
"strconv"
"strings"
@@ -26,8 +25,8 @@ func init() {
gotenberg.MustRegisterModule(API{})
}
// API is a module which provides an HTTP server. Other modules may add
// "multipart/form-data" routes, middlewares or health checks.
// API is a module which provides an HTTP server. Other modules may add routes,
// middlewares or health checks.
type API struct {
port int
readTimeout time.Duration
@@ -36,38 +35,41 @@ type API struct {
rootPath string
traceHeader string
disableHealthCheckLogging bool
webhookAllowList *regexp.Regexp
webhookDenyList *regexp.Regexp
webhookErrorAllowList *regexp.Regexp
webhookErrorDenyList *regexp.Regexp
webhookMaxRetry int
webhookRetryMinWait time.Duration
webhookRetryMaxWait time.Duration
disableWebhook bool
multipartFormDataRoutes []MultipartFormDataRoute
externalMiddlewares []Middleware
healthChecks []health.CheckerOption
logger *zap.Logger
srv *echo.Echo
routes []Route
externalMiddlewares []Middleware
healthChecks []health.CheckerOption
gcGraceDuration time.Duration
logger *zap.Logger
srv *echo.Echo
}
// MultipartFormDataRouter is a module interface which adds
// "multipart/form-data" routes to the API.
type MultipartFormDataRouter interface {
Routes() ([]MultipartFormDataRoute, error)
// Router is a module interface which adds routes to the API.
type Router interface {
Routes() ([]Route, error)
}
// MultipartFormDataRoute represents a "multipart/form-data" route. All routes
// uses the HTTP POST method.
type MultipartFormDataRoute struct {
// Route represents a route from a Router.
type Route struct {
// Method is the HTTP method of the route (i.e., GET, POST, etc.).
// Required.
Method string
// Path is the sub path of the route. Must start with a slash.
// Required.
Path string
// IsMultipart tells if the route is "multipart/form-data".
// Optional.
IsMultipart bool
// DisableLogging disables the logging for this route.
// Optional.
DisableLogging bool
// Handler is the function which handles the request.
// Required.
Handler func(ctx *Context) error
Handler echo.HandlerFunc
}
// MiddlewareProvider is a module interface which adds middlewares to the API.
@@ -75,8 +77,18 @@ type MiddlewareProvider interface {
Middlewares() ([]Middleware, error)
}
// MiddlewareStack is a type which helps to determine in which stack the
// middlewares provided by the MiddlewareProvider modules should be located.
type MiddlewareStack uint32
const (
DefaultStack MiddlewareStack = iota
PreRouterStack
MultipartStack
)
// MiddlewarePriority is a type which helps to determine the execution order of
// middlewares provided by the MiddlewareProvider modules.
// middlewares provided by the MiddlewareProvider modules in a stack.
type MiddlewarePriority uint32
const (
@@ -113,13 +125,13 @@ const (
// }(),
// }
type Middleware struct {
// RunBeforeRouter tells if the middleware should run before the router
// process an HTTP request.
// Stack tells in which stack the middleware should be located.
// Default to DefaultStack.
// Optional.
RunBeforeRouter bool
Stack MiddlewareStack
// Priority tells if the middleware should be positioned high or not in
// the middlewares chain.
// its stack.
// Default to VeryLowPriority.
// Optional.
Priority MiddlewarePriority
@@ -137,6 +149,12 @@ type HealthChecker interface {
Checks() ([]health.CheckerOption, error)
}
// GarbageCollectorGraceDurationIncrementer is a module interface for
// increasing the grace duration provided by the API for the garbage collector.
type GarbageCollectorGraceDurationIncrementer interface {
AddGraceDuration() time.Duration
}
// Descriptor returns an API's module descriptor.
func (API) Descriptor() gotenberg.ModuleDescriptor {
return gotenberg.ModuleDescriptor{
@@ -151,14 +169,6 @@ func (API) Descriptor() gotenberg.ModuleDescriptor {
fs.String("api-root-path", "/", "Set the root path of the API - for service discovery via URL paths")
fs.String("api-trace-header", "Gotenberg-Trace", "Set the header name to use for identifying requests")
fs.Bool("api-disable-health-check-logging", false, "Disable health check logging")
fs.String("api-webhook-allow-list", "", "Set the allowed URLs for the webhook feature using a regular expression")
fs.String("api-webhook-deny-list", "", "Set the denied URLs for the webhook feature using a regular expression")
fs.String("api-webhook-error-allow-list", "", "Set the allowed URLs in case of an error for the webhook feature using a regular expression")
fs.String("api-webhook-error-deny-list", "", "Set the denied URLs in case of an error for the webhook feature using a regular expression")
fs.Int("api-webhook-max-retry", 4, "Set the maximum number of retries for the webhook feature")
fs.Duration("api-webhook-retry-min-wait", time.Duration(1)*time.Second, "Set the minimum duration to wait before trying to call the webhook again")
fs.Duration("api-webhook-retry-max-wait", time.Duration(30)*time.Second, "Set the maximum duration to wait before trying to call the webhook again")
fs.Bool("api-disable-webhook", false, "Disable the webhook feature")
return fs
}(),
@@ -176,14 +186,6 @@ func (a *API) Provision(ctx *gotenberg.Context) error {
a.rootPath = flags.MustString("api-root-path")
a.traceHeader = flags.MustString("api-trace-header")
a.disableHealthCheckLogging = flags.MustBool("api-disable-health-check-logging")
a.webhookAllowList = flags.MustRegexp("api-webhook-allow-list")
a.webhookDenyList = flags.MustRegexp("api-webhook-deny-list")
a.webhookErrorAllowList = flags.MustRegexp("api-webhook-error-allow-list")
a.webhookErrorDenyList = flags.MustRegexp("api-webhook-error-deny-list")
a.webhookMaxRetry = flags.MustInt("api-webhook-max-retry")
a.webhookRetryMinWait = flags.MustDuration("api-webhook-retry-min-wait")
a.webhookRetryMaxWait = flags.MustDuration("api-webhook-retry-max-wait")
a.disableWebhook = flags.MustBool("api-disable-webhook")
// Port from env?
portEnvVar := flags.MustString("api-port-from-env")
@@ -207,14 +209,14 @@ func (a *API) Provision(ctx *gotenberg.Context) error {
}
// Get routes from modules.
mods, err := ctx.Modules(new(MultipartFormDataRouter))
mods, err := ctx.Modules(new(Router))
if err != nil {
return fmt.Errorf("get multipart/form-data routers: %w", err)
return fmt.Errorf("get routers: %w", err)
}
routers := make([]MultipartFormDataRouter, len(mods))
routers := make([]Router, len(mods))
for i, router := range mods {
routers[i] = router.(MultipartFormDataRouter)
routers[i] = router.(Router)
}
for _, router := range routers {
@@ -223,7 +225,7 @@ func (a *API) Provision(ctx *gotenberg.Context) error {
return fmt.Errorf("get routes: %w", err)
}
a.multipartFormDataRoutes = append(a.multipartFormDataRoutes, routes...)
a.routes = append(a.routes, routes...)
}
// Get middlewares from modules.
@@ -271,6 +273,18 @@ func (a *API) Provision(ctx *gotenberg.Context) error {
a.healthChecks = append(a.healthChecks, checks...)
}
// Grace duration.
a.gcGraceDuration = a.readTimeout + a.processTimeout + a.writeTimeout
mods, err = ctx.Modules(new(GarbageCollectorGraceDurationIncrementer))
if err != nil {
return fmt.Errorf("get garbage collector grace duration increments: %w", err)
}
for _, incrementer := range mods {
a.gcGraceDuration += incrementer.(GarbageCollectorGraceDurationIncrementer).AddGraceDuration()
}
loggerProvider, err := ctx.Module(new(gotenberg.LoggerProvider))
if err != nil {
return fmt.Errorf("get logger provider: %w", err)
@@ -318,26 +332,35 @@ func (a API) Validate() error {
return err
}
routesMap := make(map[string]MultipartFormDataRoute, len(a.multipartFormDataRoutes))
routesMap := make(map[string]string, len(a.routes)+1)
routesMap["/health"] = "/health"
for _, route := range a.multipartFormDataRoutes {
for _, route := range a.routes {
if route.Path == "" {
return errors.New("route with empty path cannot be registered")
}
if !strings.HasPrefix(route.Path, "/") {
return fmt.Errorf("route %s does not start with /", route.Path)
return fmt.Errorf("route '%s' does not start with /", route.Path)
}
if route.IsMultipart && !strings.HasPrefix(route.Path, "/forms") {
return fmt.Errorf("multipart/form-data route '%s' does not start with /forms", route.Path)
}
if route.Method == "" {
return fmt.Errorf("route '%s' has an empty method", route.Path)
}
if route.Handler == nil {
return fmt.Errorf("route %s has a nil handler", route.Path)
return fmt.Errorf("route '%s' has a nil handler", route.Path)
}
if _, ok := routesMap[route.Path]; ok {
return fmt.Errorf("route %s is already registered", route.Path)
return fmt.Errorf("route '%s' is already registered", route.Path)
}
routesMap[route.Path] = route
routesMap[route.Path] = route.Path
}
for _, middleware := range a.externalMiddlewares {
@@ -356,94 +379,83 @@ func (a *API) Start() error {
a.srv.HidePort = true
a.srv.Server.ReadTimeout = a.readTimeout
a.srv.Server.WriteTimeout = a.writeTimeout
a.srv.HTTPErrorHandler = httpErrorHandler(a.traceHeader)
a.srv.HTTPErrorHandler = httpErrorHandler()
// Let's prepare the modules' routes.
var disableLoggingForPaths []string
for i, route := range a.routes {
a.routes[i].Path = strings.TrimPrefix(route.Path, "/")
if route.DisableLogging {
disableLoggingForPaths = append(disableLoggingForPaths, strings.TrimPrefix(route.Path, "/"))
}
}
// Check if the user wish to add logging entries related to the health
// check route.
if a.disableHealthCheckLogging {
disableLoggingForPaths = append(disableLoggingForPaths, "health")
}
// Add the API middlewares.
a.srv.Pre(
latencyMiddleware(),
rootPathMiddleware(a.rootPath),
traceMiddleware(a.traceHeader),
loggerMiddleware(a.logger, a.disableHealthCheckLogging),
timeoutsMiddleware(a.readTimeout, a.processTimeout, a.writeTimeout),
loggerMiddleware(a.logger, disableLoggingForPaths),
)
// Add the modules' middlewares in their respective stacks.
var externalMultipartMiddlewares []Middleware
for _, externalMiddleware := range a.externalMiddlewares {
if externalMiddleware.RunBeforeRouter {
switch externalMiddleware.Stack {
case PreRouterStack:
a.srv.Pre(externalMiddleware.Handler)
continue
case MultipartStack:
externalMultipartMiddlewares = append(externalMultipartMiddlewares, externalMiddleware)
default:
a.srv.Use(externalMiddleware.Handler)
}
a.srv.Use(externalMiddleware.Handler)
}
hardTimeout := a.processTimeout + (time.Duration(5) * time.Second)
// Add the modules' routes and their specific middlewares.
for _, route := range a.routes {
var middlewares []echo.MiddlewareFunc
if route.IsMultipart {
middlewares = append(middlewares, contextMiddleware(a.processTimeout))
for _, externalMultipartMiddleware := range externalMultipartMiddlewares {
middlewares = append(middlewares, externalMultipartMiddleware.Handler)
}
}
middlewares = append(middlewares, hardTimeoutMiddleware(hardTimeout))
a.srv.Add(
route.Method,
fmt.Sprintf("%s%s", a.rootPath, route.Path),
route.Handler,
middlewares...,
)
}
// Let's not forget the health check route.
a.srv.GET(
fmt.Sprintf("%shealth", a.rootPath),
fmt.Sprintf("%s%s", a.rootPath, "health"),
func() echo.HandlerFunc {
checks := append(a.healthChecks, health.WithTimeout(a.processTimeout))
checker := health.NewChecker(checks...)
return echo.WrapHandler(health.NewHandler(checker))
}(),
timeoutMiddleware(hardTimeout),
hardTimeoutMiddleware(hardTimeout),
)
formsGroup := a.srv.Group(
fmt.Sprintf("%sforms", a.rootPath),
contextMiddleware(
contextMiddlewareConfig{
traceHeader: a.traceHeader,
timeout: struct {
process time.Duration
write time.Duration
}{
process: a.processTimeout,
write: a.writeTimeout,
},
webhook: struct {
allowList *regexp.Regexp
denyList *regexp.Regexp
errorAllowList *regexp.Regexp
errorDenyList *regexp.Regexp
maxRetry int
retryMinWait time.Duration
retryMaxWait time.Duration
disable bool
}{
allowList: a.webhookAllowList,
denyList: a.webhookDenyList,
errorAllowList: a.webhookErrorAllowList,
errorDenyList: a.webhookErrorDenyList,
maxRetry: a.webhookMaxRetry,
retryMinWait: a.webhookRetryMinWait,
retryMaxWait: a.webhookRetryMaxWait,
disable: a.disableWebhook,
},
},
),
timeoutMiddleware(hardTimeout),
)
// Add routes from other modules.
for _, route := range a.multipartFormDataRoutes {
formsGroup.POST(
route.Path,
func(route MultipartFormDataRoute) echo.HandlerFunc {
return func(c echo.Context) error {
ctx := c.Get("context").(*Context)
err := route.Handler(ctx)
if err != nil {
return fmt.Errorf("handle request: %w", err)
}
return nil
}
}(route),
)
}
// As the listen method is blocking, run it in a goroutine.
// As the following code is blocking, run it in a goroutine.
go func() {
server := &http2.Server{}
err := a.srv.StartH2CServer(fmt.Sprintf(":%d", a.port), server)
@@ -468,18 +480,7 @@ func (a API) Stop(ctx context.Context) error {
// GraceDuration updates the expiration time of files and directories parsed by
// the gc.GarbageCollector.
func (a API) GraceDuration() time.Duration {
duration := a.readTimeout + a.processTimeout + a.writeTimeout
if a.disableWebhook {
return duration
}
for i := 0; i < a.webhookMaxRetry; i++ {
// Yep... Golang does not allow int * time.Duration.
duration += a.webhookRetryMaxWait
}
return duration
return a.gcGraceDuration
}
// Interface guards.