🎉 Optimized the data preview function of log datasource

Author: Cairry

api/datasource.go

@@ -48,7 +48,7 @@ func (dc DatasourceController) API(gin *gin.RouterGroup) {
 		datasourceB.GET("dataSourceSearch", dc.Search)
 		datasourceB.GET("promQuery", dc.PromQuery)
 		datasourceB.POST("dataSourcePing", dc.Ping)
-		datasourceB.POST("esSearch", dc.EsSearch)
+		datasourceB.POST("searchViewLogsContent", dc.SearchViewLogsContent)
 	}
 
 }
@@ -180,9 +180,9 @@ func (dc DatasourceController) Ping(ctx *gin.Context) {
 	})
 }
 
-// EsSearch es 内容搜索
-func (dc DatasourceController) EsSearch(ctx *gin.Context) {
-	r := new(models.EsSearchReq)
+// SearchViewLogsContent Logs 数据预览
+func (dc DatasourceController) SearchViewLogsContent(ctx *gin.Context) {
+	r := new(models.SearchLogsContentReq)
 	BindJson(ctx, r)
 
 	Service(ctx, func() (interface{}, interface{}) {
@@ -192,45 +192,54 @@ func (dc DatasourceController) EsSearch(ctx *gin.Context) {
 		if err != nil {
 			return nil, err
 		}
+
 		datasource := data.(models.AlertDataSource)
-		client, err := provider.NewElasticSearchClient(ctx, datasource)
-		if err != nil {
-			return nil, err
-		}
+
+		var (
+			client  provider.LogsFactoryProvider
+			options provider.LogQueryOptions
+		)
 
 		// 使用 base64.StdEncoding 进行解码
 		decodedBytes, err := base64.StdEncoding.DecodeString(r.Query)
 		if err != nil {
 			return nil, fmt.Errorf("base64 解码失败: %s", err)
 		}
-
 		// 将解码后的字节转换为字符串
 		QueryStr := string(decodedBytes)
 
-		query, _, err := client.Query(provider.LogQueryOptions{ElasticSearch: provider.Elasticsearch{
-			Index:     r.GetIndexName(),
-			QueryType: "RawJson",
-			RawJson:   QueryStr,
-		}})
-		if err != nil {
-			return nil, err
-		}
+		switch r.Type {
+		case provider.VictoriaLogsDsProviderName:
+			client, err = provider.NewVictoriaLogsClient(ctx, datasource)
+			if err != nil {
+				return nil, err
+			}
 
-		type newLogStruct struct {
-			Metric  map[string]interface{}
-			Message interface{}
-		}
+			options = provider.LogQueryOptions{
+				VictoriaLogs: provider.VictoriaLogs{
+					Query: QueryStr,
+				},
+			}
+		case provider.ElasticSearchDsProviderName:
+			client, err = provider.NewElasticSearchClient(ctx, datasource)
+			if err != nil {
+				return nil, err
+			}
 
-		var newData []newLogStruct
-		for _, v := range query {
-			for _, message := range v.Message {
-				newData = append(newData, newLogStruct{
-					Metric:  v.Metric,
-					Message: message,
-				})
+			options = provider.LogQueryOptions{
+				ElasticSearch: provider.Elasticsearch{
+					Index:     r.GetElasticSearchIndexName(),
+					QueryType: "RawJson",
+					RawJson:   QueryStr,
+				},
 			}
 		}
 
-		return tools.JsonMarshal(newData), nil
+		query, _, err := client.Query(options)
+		if err != nil {
+			return nil, err
+		}
+
+		return query, nil
 	})
 }

internal/models/datasource.go

@@ -77,24 +77,21 @@ func (d *AlertDataSource) GetEnabled() *bool {
 	return d.Enabled
 }
 
-type EsSearchReq struct {
+type SearchLogsContentReq struct {
+	Type         string `json:"type"`
 	DatasourceId string `json:"datasourceId"`
 	Index        string `json:"index"`
 	Query        string `json:"query"`
 }
 
-func (e EsSearchReq) GetIndexName() string {
-	if strings.Contains(e.Index, "YYYY") && strings.Contains(e.Index, "MM") && strings.Contains(e.Index, "dd") {
-		indexName := e.Index
+func (s SearchLogsContentReq) GetElasticSearchIndexName() string {
+	if strings.Contains(s.Index, "YYYY") && strings.Contains(s.Index, "MM") && strings.Contains(s.Index, "dd") {
+		indexName := s.Index
 		indexName = strings.ReplaceAll(indexName, "YYYY", time.Now().Format("2006"))
 		indexName = strings.ReplaceAll(indexName, "MM", time.Now().Format("01"))
 		indexName = strings.ReplaceAll(indexName, "dd", time.Now().Format("02"))
 		return indexName
 	}
 
-	return e.Index
-}
-
-type EsSearchRes struct {
-	Data data `json:"data"`
+	return s.Index
 }

internal/models/user_permissions.go

@@ -435,9 +435,9 @@ func PermissionsInfo() map[string]UserPermissions {
 			Key: "修改故障中心基本信息",
 			API: "/api/w8t/faultCenter/faultCenterReset",
 		},
-		"esSearch": {
-			Key: "ElasticSearch搜索",
-			API: "/api/w8t/datasource/esSearch",
+		"searchViewLogsContent": {
+			Key: "搜索VictoriaLogs数据源内容",
+			API: "/api/w8t/datasource/searchViewLogsContent",
 		},
 	}
 }

internal/services/dashboard.go

@@ -168,14 +168,15 @@ func (ds dashboardService) ListGrafanaDashboards(req interface{}) (data interfac
 		return nil, fmt.Errorf("invalid grafana version, please change v10 or v11")
 	}
 
-	get, err := tools.Get(nil, fmt.Sprintf("%s/api/search?%s", r.GrafanaHost, query), 10)
+	requestURL := fmt.Sprintf("%s/api/search?%s", r.GrafanaHost, query)
+	get, err := tools.Get(nil, requestURL, 10)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("请求错误, err: %s", err.Error())
 	}
 
 	var d []models.GrafanaDashboardInfo
 	if err := tools.ParseReaderBody(get.Body, &d); err != nil {
-		return nil, err
+		return nil, fmt.Errorf("读取body错误, err: %s", err.Error())
 	}
 
 	return d, nil

internal/services/datasource.go

@@ -127,7 +127,7 @@ func (ds datasourceService) WithAddClientToProviderPools(datasource models.Alert
 	case provider.ElasticSearchDsProviderName:
 		cli, err = provider.NewElasticSearchClient(ctx.Ctx, datasource)
 	case provider.VictoriaLogsDsProviderName:
-		cli, err = provider.NewVictoriaLogsProvider(ctx.Ctx, datasource)
+		cli, err = provider.NewVictoriaLogsClient(ctx.Ctx, datasource)
 	case provider.JaegerDsProviderName:
 		cli, err = provider.NewJaegerClient(datasource)
 	case "Kubernetes":

pkg/provider/check.go

@@ -42,7 +42,7 @@ var datasourceFactories = map[string]ClientFactory{
 		return &CloudWatchDummyChecker{}, nil
 	},
 	"VictoriaLogs": func(ds models.AlertDataSource) (HealthChecker, error) {
-		return NewVictoriaLogsClient(ds)
+		return NewVictoriaLogsClient(context.Background(), ds)
 	},
 }
 

pkg/provider/logs_victoria.go

@@ -9,7 +9,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"strconv"
 	"time"
 	"watchAlert/pkg/ctx"
 
@@ -31,34 +30,26 @@ type (
 )
 
 // NewVictoriaLogsClient 创建一个新的 VictoriaLogsProvider 实例。
-func NewVictoriaLogsClient(ds models.AlertDataSource) (VictoriaLogsProvider, error) {
-	return VictoriaLogsProvider{
-		URL:            ds.HTTP.URL,
-		ExternalLabels: ds.Labels,
-		Username:       ds.Auth.User,
-		Password:       ds.Auth.Pass,
-	}, nil
-}
-
-func NewVictoriaLogsProvider(ctx context.Context, datasource models.AlertDataSource) (LogsFactoryProvider, error) {
+func NewVictoriaLogsClient(ctx context.Context, datasource models.AlertDataSource) (LogsFactoryProvider, error) {
 	return VictoriaLogsProvider{
 		URL:            datasource.HTTP.URL,
 		Timeout:        datasource.HTTP.Timeout,
 		ExternalLabels: datasource.Labels,
+		Username:       datasource.Auth.User,
+		Password:       datasource.Auth.Pass,
 		Ctx:            ctx,
 	}, nil
 }
 
 func (v VictoriaLogsProvider) Query(options LogQueryOptions) ([]Logs, int, error) {
 	curTime := time.Now()
 
-	if options.StartAt == "" {
-		duration, _ := time.ParseDuration(strconv.Itoa(1) + "h")
-		options.StartAt = curTime.Add(-duration).Format(time.RFC3339Nano)
+	if options.StartAt == "" || options.StartAt == nil {
+		options.StartAt = int32(tools.ParserDuration(curTime, 30, "m").Unix())
 	}
 
-	if options.EndAt == "" {
-		options.EndAt = curTime.Format(time.RFC3339Nano)
+	if options.EndAt == "" || options.EndAt == nil {
+		options.EndAt = int32(curTime.Unix())
 	}
 
 	if options.VictoriaLogs.Limit == 0 {